From 6fd3c180f284c89ccccada32de75be541b982326 Mon Sep 17 00:00:00 2001 From: sds100 Date: Tue, 1 Apr 2025 23:48:20 -0600 Subject: [PATCH 01/21] #257 create accessibility node repository and WIP recording of nodes --- .../github/sds100/keymapper/ServiceLocator.kt | 15 +++ .../data/entities/AccessibilityNodeEntity.kt | 16 +++ .../AccessibilityNodeRepository.kt | 34 +++++++ .../AccessibilityNodeRecorder.kt | 74 ++++++++++++++ .../BaseAccessibilityServiceController.kt | 99 +++++++++++++------ .../accessibility/MyAccessibilityService.kt | 13 ++- .../sds100/keymapper/util/ServiceEvent.kt | 6 ++ 7 files changed, 225 insertions(+), 32 deletions(-) create mode 100644 app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt diff --git a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt index 00f14520f2..342bd175ce 100755 --- a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt +++ b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt @@ -8,6 +8,8 @@ import io.github.sds100.keymapper.actions.sound.SoundsManagerImpl import io.github.sds100.keymapper.backup.BackupManager import io.github.sds100.keymapper.backup.BackupManagerImpl import io.github.sds100.keymapper.data.db.AppDatabase +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepositoryImpl import io.github.sds100.keymapper.data.repositories.FloatingButtonRepository import io.github.sds100.keymapper.data.repositories.FloatingLayoutRepository import io.github.sds100.keymapper.data.repositories.GroupRepository @@ -214,6 +216,19 @@ object ServiceLocator { ) } + @Volatile + private var accessibilityNodeRepository: AccessibilityNodeRepository? = null + + fun accessibilityNodeRepository(context: Context): AccessibilityNodeRepository { + synchronized(this) { + return accessibilityNodeRepository ?: AccessibilityNodeRepositoryImpl( + (context.applicationContext as KeyMapperApp).appCoroutineScope, + ).also { + this.accessibilityNodeRepository = it + } + } + } + fun fileAdapter(context: Context): FileAdapter = (context.applicationContext as KeyMapperApp).fileAdapter fun inputMethodAdapter(context: Context): InputMethodAdapter = (context.applicationContext as KeyMapperApp).inputMethodAdapter diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt new file mode 100644 index 0000000000..863983b572 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -0,0 +1,16 @@ +package io.github.sds100.keymapper.data.entities + +data class AccessibilityNodeEntity( + val id: Long = 0L, + val parentId: Long? = null, + val packageName: String, + val text: String?, + val contentDescription: String?, + val className: String?, + val viewResourceId: String?, + val uniqueId: String?, + /** + * A list of the allowed accessibility node actions. + */ + val actions: List, +) diff --git a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt new file mode 100644 index 0000000000..b2fa1e522f --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt @@ -0,0 +1,34 @@ +package io.github.sds100.keymapper.data.repositories + +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.util.State +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.launch + +interface AccessibilityNodeRepository { + val nodes: Flow>> + fun insert(vararg node: AccessibilityNodeEntity) + fun deleteAll() +} + +class AccessibilityNodeRepositoryImpl(private val coroutineScope: CoroutineScope) : AccessibilityNodeRepository { + override val nodes = + MutableStateFlow>>(State.Data(ArrayList(128))) + + override fun insert(vararg node: AccessibilityNodeEntity) { + coroutineScope.launch { + val currentState = nodes.value + if (currentState is State.Data) { + nodes.emit(State.Data(currentState.data.plus(node))) + } + } + } + + override fun deleteAll() { + coroutineScope.launch { + nodes.emit(State.Data(ArrayList(128))) + } + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt new file mode 100644 index 0000000000..71cb54bcf2 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -0,0 +1,74 @@ +package io.github.sds100.keymapper.system.accessibility + +import android.accessibilityservice.AccessibilityService +import android.os.Build +import android.view.accessibility.AccessibilityEvent +import io.github.sds100.keymapper.ServiceLocator +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +class AccessibilityNodeRecorder( + private val service: AccessibilityService, + private val coroutineScope: CoroutineScope, +) { + companion object { + private const val RECORD_DURATION = 60000L + } + + private val nodeRepository: AccessibilityNodeRepository by lazy { + ServiceLocator.accessibilityNodeRepository(service) + } + + private var recordJob: Job? = null + private val _isRecording = MutableStateFlow(false) + val isRecording = _isRecording.asStateFlow() + + fun startRecording() { + _isRecording.update { true } + recordJob?.cancel() + recordJob = recordJob() + } + + fun stopRecording() { + recordJob?.cancel() + recordJob = null + _isRecording.update { false } + } + + fun onAccessibilityEvent(event: AccessibilityEvent) { + if (!isRecording.value) { + return + } + + val source = event.source ?: return + + val entity = + AccessibilityNodeEntity( + packageName = event.packageName.toString(), + text = source.text.firstOrNull()?.toString(), + contentDescription = source.contentDescription?.toString(), + className = source.className?.toString(), + viewResourceId = source.viewIdResourceName, + uniqueId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + source.uniqueId + } else { + null + }, + actions = source.actionList?.map { it.id } ?: emptyList(), + ) + + nodeRepository.insert(entity) + } + + private fun recordJob() = coroutineScope.launch { + delay(RECORD_DURATION) + _isRecording.update { false } + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt index 6da22df6af..676c6ea7ee 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt @@ -60,7 +60,7 @@ import timber.log.Timber */ abstract class BaseAccessibilityServiceController( private val coroutineScope: CoroutineScope, - private val accessibilityService: IAccessibilityService, + private val service: MyAccessibilityService, private val inputEvents: SharedFlow, private val outputEvents: MutableSharedFlow, private val detectConstraintsUseCase: DetectConstraintsUseCase, @@ -101,6 +101,9 @@ abstract class BaseAccessibilityServiceController( rerouteKeyEventsUseCase, ) + private val accessibilityNodeRecorder: AccessibilityNodeRecorder = + AccessibilityNodeRecorder(service, coroutineScope) + private var recordingTriggerJob: Job? = null private val recordingTrigger: Boolean get() = recordingTriggerJob != null && recordingTriggerJob?.isActive == true @@ -114,7 +117,7 @@ abstract class BaseAccessibilityServiceController( detectKeyMapsUseCase.detectScreenOffTriggers .stateIn(coroutineScope, SharingStarted.Eagerly, false) - private val changeImeOnInputFocus: StateFlow = + private val changeImeOnInputFocusFlow: StateFlow = settingsRepository .get(Keys.changeImeOnInputFocus) .map { it ?: PreferenceDefaults.CHANGE_IME_ON_INPUT_FOCUS } @@ -145,6 +148,7 @@ abstract class BaseAccessibilityServiceController( // This is required for receive TYPE_WINDOWS_CHANGED events so can // detect when to show/hide overlays. .withFlag(AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS) + .withFlag(AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { flags = flags.withFlag(AccessibilityServiceInfo.FLAG_ENABLE_ACCESSIBILITY_VOLUME) @@ -171,24 +175,25 @@ abstract class BaseAccessibilityServiceController( MutableStateFlow(AccessibilityEvent.TYPE_WINDOWS_CHANGED) init { + serviceFlags.onEach { flags -> // check that it isn't null because this can only be called once the service is bound - if (accessibilityService.serviceFlags != null) { - accessibilityService.serviceFlags = flags + if (service.serviceFlags != null) { + service.serviceFlags = flags } }.launchIn(coroutineScope) serviceFeedbackType.onEach { feedbackType -> // check that it isn't null because this can only be called once the service is bound - if (accessibilityService.serviceFeedbackType != null) { - accessibilityService.serviceFeedbackType = feedbackType + if (service.serviceFeedbackType != null) { + service.serviceFeedbackType = feedbackType } }.launchIn(coroutineScope) serviceEventTypes.onEach { eventTypes -> // check that it isn't null because this can only be called once the service is bound - if (accessibilityService.serviceEventTypes != null) { - accessibilityService.serviceEventTypes = eventTypes + if (service.serviceEventTypes != null) { + service.serviceEventTypes = eventTypes } }.launchIn(coroutineScope) @@ -224,7 +229,7 @@ abstract class BaseAccessibilityServiceController( onEventFromUi(it) }.launchIn(coroutineScope) - accessibilityService.isKeyboardHidden + service.isKeyboardHidden .drop(1) // Don't send it when collecting initially .onEach { isHidden -> if (isHidden) { @@ -255,23 +260,44 @@ abstract class BaseAccessibilityServiceController( } }.launchIn(coroutineScope) - changeImeOnInputFocus.onEach { changeImeOnInputFocus -> - if (changeImeOnInputFocus) { - serviceEventTypes.value = serviceEventTypes.value - .withFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) - .withFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) - } else { - serviceEventTypes.value = serviceEventTypes.value - .minusFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) - .minusFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) - } - }.launchIn(coroutineScope) + val imeInputFocusEvents = + AccessibilityEvent.TYPE_VIEW_FOCUSED or AccessibilityEvent.TYPE_VIEW_CLICKED + val recordNodeEvents = AccessibilityEvent.TYPE_VIEW_FOCUSED or + AccessibilityEvent.TYPE_VIEW_CLICKED or + AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED + +// coroutineScope.launch { +// combine( +// changeImeOnInputFocusFlow, +// accessibilityNodeRecorder.isRecording +// ) { changeImeOnInputFocus, isRecordingNodes -> +// serviceEventTypes.update { eventTypes -> +// var newEventTypes = eventTypes +// +// +// +// newEventTypes +// } +// +// }.collect() +// } +// changeImeOnInputFocusFlow.onEach { changeImeOnInputFocus -> +// if (changeImeOnInputFocus) { +// serviceEventTypes.value = serviceEventTypes.value +// .withFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) +// .withFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) +// } else { +// serviceEventTypes.value = serviceEventTypes.value +// .minusFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) +// .minusFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) +// } +// }.launchIn(coroutineScope) } open fun onServiceConnected() { - accessibilityService.serviceFlags = serviceFlags.value - accessibilityService.serviceFeedbackType = serviceFeedbackType.value - accessibilityService.serviceEventTypes = serviceEventTypes.value + service.serviceFlags = serviceFlags.value + service.serviceFeedbackType = serviceFeedbackType.value + service.serviceEventTypes = serviceEventTypes.value // check if fingerprint gestures are supported if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { @@ -284,7 +310,7 @@ abstract class BaseAccessibilityServiceController( * used while this is called. */ if (fingerprintGesturesSupported.isSupported.firstBlocking() != true) { fingerprintGesturesSupported.setSupported( - accessibilityService.isFingerprintGestureDetectionAvailable, + service.isFingerprintGestureDetectionAvailable, ) } @@ -441,10 +467,12 @@ abstract class BaseAccessibilityServiceController( } } - open fun onAccessibilityEvent(event: AccessibilityEventModel) { - if (changeImeOnInputFocus.value) { + open fun onAccessibilityEvent(event: AccessibilityEvent) { + accessibilityNodeRecorder.onAccessibilityEvent(event) + + if (changeImeOnInputFocusFlow.value) { val focussedNode = - accessibilityService.findFocussedNode(AccessibilityNodeInfo.FOCUS_INPUT) + service.findFocussedNode(AccessibilityNodeInfo.FOCUS_INPUT) if (focussedNode?.isEditable == true && focussedNode.isFocused) { Timber.d("Got input focus") @@ -497,14 +525,23 @@ abstract class BaseAccessibilityServiceController( outputEvents.emit(ServiceEvent.Pong(event.key)) } - is ServiceEvent.HideKeyboard -> accessibilityService.hideKeyboard() - is ServiceEvent.ShowKeyboard -> accessibilityService.showKeyboard() - is ServiceEvent.ChangeIme -> accessibilityService.switchIme(event.imeId) + is ServiceEvent.HideKeyboard -> service.hideKeyboard() + is ServiceEvent.ShowKeyboard -> service.showKeyboard() + is ServiceEvent.ChangeIme -> service.switchIme(event.imeId) is ServiceEvent.DisableService -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - accessibilityService.disableSelf() + service.disableSelf() } is ServiceEvent.TriggerKeyMap -> triggerKeyMapFromIntent(event.uid) + + is ServiceEvent.StartRecordingNodes -> { + accessibilityNodeRecorder.startRecording() + } + + is ServiceEvent.StopRecordingNodes -> { + accessibilityNodeRecorder.stopRecording() + } + else -> Unit } } diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt index b909f9ae6f..836166c4e5 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt @@ -21,7 +21,9 @@ import androidx.lifecycle.LifecycleRegistry import androidx.savedstate.SavedStateRegistry import androidx.savedstate.SavedStateRegistryController import androidx.savedstate.SavedStateRegistryOwner +import io.github.sds100.keymapper.Constants import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.ServiceLocator import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType import io.github.sds100.keymapper.api.IKeyEventRelayServiceCallback import io.github.sds100.keymapper.api.KeyEventRelayService @@ -37,6 +39,7 @@ import io.github.sds100.keymapper.util.InputEventType import io.github.sds100.keymapper.util.MathUtils import io.github.sds100.keymapper.util.Result import io.github.sds100.keymapper.util.Success +import io.github.sds100.keymapper.util.onSuccess import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.update @@ -187,6 +190,14 @@ class MyAccessibilityService : override fun onServiceConnected() { super.onServiceConnected() + val inputMethodAdapter = ServiceLocator.inputMethodAdapter(this) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + inputMethodAdapter.getInfoByPackageName(Constants.PACKAGE_NAME).onSuccess { + softKeyboardController.setInputMethodEnabled(it.id, true) + softKeyboardController.switchToInputMethod(it.id) + } + } Timber.i("Accessibility service: onServiceConnected") lifecycleRegistry.currentState = Lifecycle.State.STARTED @@ -281,7 +292,7 @@ class MyAccessibilityService : _activeWindowPackage.update { rootInActiveWindow?.packageName?.toString() } } - controller?.onAccessibilityEvent(event.toModel()) + controller?.onAccessibilityEvent(event) } override fun onKeyEvent(event: KeyEvent?): Boolean { diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt b/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt index 4db23d48bb..20777f5e4f 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt @@ -69,4 +69,10 @@ sealed class ServiceEvent { @Serializable data class TriggerKeyMap(val uid: String) : ServiceEvent() + + @Serializable + data object StartRecordingNodes : ServiceEvent() + + @Serializable + data object StopRecordingNodes : ServiceEvent() } From 4660c001e17011d5555aafebdb3db2ba6b4bfc0d Mon Sep 17 00:00:00 2001 From: sds100 Date: Tue, 1 Apr 2025 23:53:48 -0600 Subject: [PATCH 02/21] #257 add field for whether the user interacted with the node --- .../keymapper/data/entities/AccessibilityNodeEntity.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt index 863983b572..f480ef56e6 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -13,4 +13,10 @@ data class AccessibilityNodeEntity( * A list of the allowed accessibility node actions. */ val actions: List, + /** + * The accessibility action id of how the user interacted + * with this node. This is null if the user didn't interact with + * this node. + */ + val userInteractedActionId: Int?, ) From d05986480b2e51c708e7e30ed0a104486baf9172 Mon Sep 17 00:00:00 2001 From: sds100 Date: Wed, 30 Apr 2025 09:09:28 +0200 Subject: [PATCH 03/21] #257 WIP: create interact with UI element action --- .../sds100/keymapper/actions/ActionData.kt | 147 ++++++++------- .../actions/ActionDataEntityMapper.kt | 173 +++++++++++++++++- .../sds100/keymapper/actions/ActionId.kt | 1 + .../keymapper/actions/ActionUiHelper.kt | 2 + .../sds100/keymapper/actions/ActionUtils.kt | 6 + .../keymapper/data/entities/ActionEntity.kt | 17 ++ .../AccessibilityNodeRepository.kt | 1 + .../accessibility/AccessibilityNodeModel.kt | 15 ++ .../accessibility/AccessibilityUtils.kt | 9 + .../util/ui/compose/icons/JumpToElement.kt | 115 ++++++++++++ app/src/main/res/values/strings.xml | 41 ++++- 11 files changed, 450 insertions(+), 77 deletions(-) create mode 100644 app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt index 5f4799294d..38a2587ffc 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt @@ -1,6 +1,7 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType +import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeModel import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.display.Orientation import io.github.sds100.keymapper.system.intents.IntentExtraModel @@ -260,7 +261,7 @@ sealed class ActionData : Comparable { } @Serializable - object Disable : DoNotDisturb() { + data object Disable : DoNotDisturb() { override val id = ActionId.DISABLE_DND_MODE } } @@ -268,32 +269,32 @@ sealed class ActionData : Comparable { @Serializable sealed class Rotation : ActionData() { @Serializable - object EnableAuto : Rotation() { + data object EnableAuto : Rotation() { override val id = ActionId.ENABLE_AUTO_ROTATE } @Serializable - object DisableAuto : Rotation() { + data object DisableAuto : Rotation() { override val id = ActionId.DISABLE_AUTO_ROTATE } @Serializable - object ToggleAuto : Rotation() { + data object ToggleAuto : Rotation() { override val id = ActionId.TOGGLE_AUTO_ROTATE } @Serializable - object Portrait : Rotation() { + data object Portrait : Rotation() { override val id = ActionId.PORTRAIT_MODE } @Serializable - object Landscape : Rotation() { + data object Landscape : Rotation() { override val id = ActionId.LANDSCAPE_MODE } @Serializable - object SwitchOrientation : Rotation() { + data object SwitchOrientation : Rotation() { override val id = ActionId.SWITCH_ORIENTATION } @@ -369,37 +370,37 @@ sealed class ActionData : Comparable { @Serializable sealed class ControlMedia : ActionData() { @Serializable - object Pause : ControlMedia() { + data object Pause : ControlMedia() { override val id = ActionId.PAUSE_MEDIA } @Serializable - object Play : ControlMedia() { + data object Play : ControlMedia() { override val id = ActionId.PLAY_MEDIA } @Serializable - object PlayPause : ControlMedia() { + data object PlayPause : ControlMedia() { override val id = ActionId.PLAY_PAUSE_MEDIA } @Serializable - object NextTrack : ControlMedia() { + data object NextTrack : ControlMedia() { override val id = ActionId.NEXT_TRACK } @Serializable - object PreviousTrack : ControlMedia() { + data object PreviousTrack : ControlMedia() { override val id = ActionId.PREVIOUS_TRACK } @Serializable - object FastForward : ControlMedia() { + data object FastForward : ControlMedia() { override val id = ActionId.FAST_FORWARD } @Serializable - object Rewind : ControlMedia() { + data object Rewind : ControlMedia() { override val id = ActionId.REWIND } } @@ -537,17 +538,17 @@ sealed class ActionData : Comparable { @Serializable sealed class Wifi : ActionData() { @Serializable - object Enable : Wifi() { + data object Enable : Wifi() { override val id = ActionId.ENABLE_WIFI } @Serializable - object Disable : Wifi() { + data object Disable : Wifi() { override val id = ActionId.DISABLE_WIFI } @Serializable - object Toggle : Wifi() { + data object Toggle : Wifi() { override val id = ActionId.TOGGLE_WIFI } } @@ -555,17 +556,17 @@ sealed class ActionData : Comparable { @Serializable sealed class Bluetooth : ActionData() { @Serializable - object Enable : Bluetooth() { + data object Enable : Bluetooth() { override val id = ActionId.ENABLE_BLUETOOTH } @Serializable - object Disable : Bluetooth() { + data object Disable : Bluetooth() { override val id = ActionId.DISABLE_BLUETOOTH } @Serializable - object Toggle : Bluetooth() { + data object Toggle : Bluetooth() { override val id = ActionId.TOGGLE_BLUETOOTH } } @@ -573,17 +574,17 @@ sealed class ActionData : Comparable { @Serializable sealed class Nfc : ActionData() { @Serializable - object Enable : Nfc() { + data object Enable : Nfc() { override val id = ActionId.ENABLE_NFC } @Serializable - object Disable : Nfc() { + data object Disable : Nfc() { override val id = ActionId.DISABLE_NFC } @Serializable - object Toggle : Nfc() { + data object Toggle : Nfc() { override val id = ActionId.TOGGLE_NFC } } @@ -591,17 +592,17 @@ sealed class ActionData : Comparable { @Serializable sealed class AirplaneMode : ActionData() { @Serializable - object Enable : AirplaneMode() { + data object Enable : AirplaneMode() { override val id = ActionId.ENABLE_AIRPLANE_MODE } @Serializable - object Disable : AirplaneMode() { + data object Disable : AirplaneMode() { override val id = ActionId.DISABLE_AIRPLANE_MODE } @Serializable - object Toggle : AirplaneMode() { + data object Toggle : AirplaneMode() { override val id = ActionId.TOGGLE_AIRPLANE_MODE } } @@ -609,17 +610,17 @@ sealed class ActionData : Comparable { @Serializable sealed class MobileData : ActionData() { @Serializable - object Enable : MobileData() { + data object Enable : MobileData() { override val id = ActionId.ENABLE_MOBILE_DATA } @Serializable - object Disable : MobileData() { + data object Disable : MobileData() { override val id = ActionId.DISABLE_MOBILE_DATA } @Serializable - object Toggle : MobileData() { + data object Toggle : MobileData() { override val id = ActionId.TOGGLE_MOBILE_DATA } } @@ -627,27 +628,27 @@ sealed class ActionData : Comparable { @Serializable sealed class Brightness : ActionData() { @Serializable - object EnableAuto : Brightness() { + data object EnableAuto : Brightness() { override val id = ActionId.ENABLE_AUTO_BRIGHTNESS } @Serializable - object DisableAuto : Brightness() { + data object DisableAuto : Brightness() { override val id = ActionId.DISABLE_AUTO_BRIGHTNESS } @Serializable - object ToggleAuto : Brightness() { + data object ToggleAuto : Brightness() { override val id = ActionId.TOGGLE_AUTO_BRIGHTNESS } @Serializable - object Increase : Brightness() { + data object Increase : Brightness() { override val id = ActionId.INCREASE_BRIGHTNESS } @Serializable - object Decrease : Brightness() { + data object Decrease : Brightness() { override val id = ActionId.DECREASE_BRIGHTNESS } } @@ -655,178 +656,178 @@ sealed class ActionData : Comparable { @Serializable sealed class StatusBar : ActionData() { @Serializable - object ExpandNotifications : StatusBar() { + data object ExpandNotifications : StatusBar() { override val id = ActionId.EXPAND_NOTIFICATION_DRAWER } @Serializable - object ToggleNotifications : StatusBar() { + data object ToggleNotifications : StatusBar() { override val id = ActionId.TOGGLE_NOTIFICATION_DRAWER } @Serializable - object ExpandQuickSettings : StatusBar() { + data object ExpandQuickSettings : StatusBar() { override val id = ActionId.EXPAND_QUICK_SETTINGS } @Serializable - object ToggleQuickSettings : StatusBar() { + data object ToggleQuickSettings : StatusBar() { override val id = ActionId.TOGGLE_QUICK_SETTINGS } @Serializable - object Collapse : StatusBar() { + data object Collapse : StatusBar() { override val id = ActionId.COLLAPSE_STATUS_BAR } } @Serializable - object GoBack : ActionData() { + data object GoBack : ActionData() { override val id = ActionId.GO_BACK } @Serializable - object GoHome : ActionData() { + data object GoHome : ActionData() { override val id = ActionId.GO_HOME } @Serializable - object OpenRecents : ActionData() { + data object OpenRecents : ActionData() { override val id = ActionId.OPEN_RECENTS } @Serializable - object GoLastApp : ActionData() { + data object GoLastApp : ActionData() { override val id = ActionId.GO_LAST_APP } @Serializable - object OpenMenu : ActionData() { + data object OpenMenu : ActionData() { override val id = ActionId.OPEN_MENU } @Serializable - object ToggleSplitScreen : ActionData() { + data object ToggleSplitScreen : ActionData() { override val id = ActionId.TOGGLE_SPLIT_SCREEN } @Serializable - object Screenshot : ActionData() { + data object Screenshot : ActionData() { override val id = ActionId.SCREENSHOT } @Serializable - object MoveCursorToEnd : ActionData() { + data object MoveCursorToEnd : ActionData() { override val id = ActionId.MOVE_CURSOR_TO_END } @Serializable - object ToggleKeyboard : ActionData() { + data object ToggleKeyboard : ActionData() { override val id = ActionId.TOGGLE_KEYBOARD } @Serializable - object ShowKeyboard : ActionData() { + data object ShowKeyboard : ActionData() { override val id = ActionId.SHOW_KEYBOARD } @Serializable - object HideKeyboard : ActionData() { + data object HideKeyboard : ActionData() { override val id = ActionId.HIDE_KEYBOARD } @Serializable - object ShowKeyboardPicker : ActionData() { + data object ShowKeyboardPicker : ActionData() { override val id = ActionId.SHOW_KEYBOARD_PICKER } @Serializable - object CopyText : ActionData() { + data object CopyText : ActionData() { override val id = ActionId.TEXT_COPY } @Serializable - object PasteText : ActionData() { + data object PasteText : ActionData() { override val id = ActionId.TEXT_PASTE } @Serializable - object CutText : ActionData() { + data object CutText : ActionData() { override val id = ActionId.TEXT_CUT } @Serializable - object SelectWordAtCursor : ActionData() { + data object SelectWordAtCursor : ActionData() { override val id = ActionId.SELECT_WORD_AT_CURSOR } @Serializable - object VoiceAssistant : ActionData() { + data object VoiceAssistant : ActionData() { override val id = ActionId.OPEN_VOICE_ASSISTANT } @Serializable - object DeviceAssistant : ActionData() { + data object DeviceAssistant : ActionData() { override val id = ActionId.OPEN_DEVICE_ASSISTANT } @Serializable - object OpenCamera : ActionData() { + data object OpenCamera : ActionData() { override val id = ActionId.OPEN_CAMERA } @Serializable - object LockDevice : ActionData() { + data object LockDevice : ActionData() { override val id = ActionId.LOCK_DEVICE } @Serializable - object ScreenOnOff : ActionData() { + data object ScreenOnOff : ActionData() { override val id = ActionId.POWER_ON_OFF_DEVICE } @Serializable - object SecureLock : ActionData() { + data object SecureLock : ActionData() { override val id = ActionId.SECURE_LOCK_DEVICE } @Serializable - object ConsumeKeyEvent : ActionData() { + data object ConsumeKeyEvent : ActionData() { override val id = ActionId.CONSUME_KEY_EVENT } @Serializable - object OpenSettings : ActionData() { + data object OpenSettings : ActionData() { override val id = ActionId.OPEN_SETTINGS } @Serializable - object ShowPowerMenu : ActionData() { + data object ShowPowerMenu : ActionData() { override val id = ActionId.SHOW_POWER_MENU } @Serializable - object DismissLastNotification : ActionData() { + data object DismissLastNotification : ActionData() { override val id: ActionId = ActionId.DISMISS_MOST_RECENT_NOTIFICATION } @Serializable - object DismissAllNotifications : ActionData() { + data object DismissAllNotifications : ActionData() { override val id: ActionId = ActionId.DISMISS_ALL_NOTIFICATIONS } @Serializable - object AnswerCall : ActionData() { + data object AnswerCall : ActionData() { override val id: ActionId = ActionId.ANSWER_PHONE_CALL } @Serializable - object EndCall : ActionData() { + data object EndCall : ActionData() { override val id: ActionId = ActionId.END_PHONE_CALL } @Serializable - object DeviceControls : ActionData() { + data object DeviceControls : ActionData() { override val id: ActionId = ActionId.DEVICE_CONTROLS } @@ -845,4 +846,12 @@ sealed class ActionData : Comparable { return "HttpRequest(description=$description)" } } + + @Serializable + data class InteractUiElement( + val nodeAction: Int, + val node: AccessibilityNodeModel, + ) : ActionData() { + override val id: ActionId = ActionId.INTERACT_UI_ELEMENT + } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index 53a6515254..063dd5d8c2 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -5,6 +5,7 @@ import io.github.sds100.keymapper.data.db.typeconverter.ConstantTypeConverters import io.github.sds100.keymapper.data.entities.ActionEntity import io.github.sds100.keymapper.data.entities.EntityExtra import io.github.sds100.keymapper.data.entities.getData +import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeModel import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.intents.IntentExtraModel import io.github.sds100.keymapper.system.intents.IntentTarget @@ -12,6 +13,7 @@ import io.github.sds100.keymapper.system.network.HttpMethod import io.github.sds100.keymapper.system.volume.DndMode import io.github.sds100.keymapper.system.volume.RingerMode import io.github.sds100.keymapper.system.volume.VolumeStream +import io.github.sds100.keymapper.util.Success import io.github.sds100.keymapper.util.getKey import io.github.sds100.keymapper.util.success import io.github.sds100.keymapper.util.then @@ -41,6 +43,8 @@ object ActionDataEntityMapper { ActionEntity.Type.SYSTEM_ACTION -> { SYSTEM_ACTION_ID_MAP.getKey(entity.data) ?: return null } + + ActionEntity.Type.INTERACT_UI_ELEMENT -> ActionId.INTERACT_UI_ELEMENT } return when (actionId) { @@ -266,7 +270,7 @@ object ActionDataEntityMapper { ActionId.VOLUME_TOGGLE_MUTE, ActionId.VOLUME_UNMUTE, ActionId.VOLUME_MUTE, - -> { + -> { val showVolumeUi = entity.flags.hasFlag(ActionEntity.ACTION_FLAG_SHOW_VOLUME_UI) @@ -287,7 +291,7 @@ object ActionDataEntityMapper { ActionId.TOGGLE_FLASHLIGHT, ActionId.ENABLE_FLASHLIGHT, ActionId.CHANGE_FLASHLIGHT_STRENGTH, - -> { + -> { val lens = entity.extras.getData(ActionEntity.EXTRA_LENS).then { LENS_MAP.getKey(it)!!.success() }.valueOrNull() ?: return null @@ -313,7 +317,7 @@ object ActionDataEntityMapper { } ActionId.DISABLE_FLASHLIGHT, - -> { + -> { val lens = entity.extras.getData(ActionEntity.EXTRA_LENS).then { LENS_MAP.getKey(it)!!.success() }.valueOrNull() ?: return null @@ -322,7 +326,7 @@ object ActionDataEntityMapper { ActionId.TOGGLE_DND_MODE, ActionId.ENABLE_DND_MODE, - -> { + -> { val dndMode = entity.extras.getData(ActionEntity.EXTRA_DND_MODE).then { DND_MODE_MAP.getKey(it)!!.success() }.valueOrNull() ?: return null @@ -349,7 +353,7 @@ object ActionDataEntityMapper { ActionId.PREVIOUS_TRACK_PACKAGE, ActionId.FAST_FORWARD_PACKAGE, ActionId.REWIND_PACKAGE, - -> { + -> { val packageName = entity.extras.getData(ActionEntity.EXTRA_PACKAGE_NAME).valueOrNull() ?: return null @@ -522,6 +526,70 @@ object ActionDataEntityMapper { authorizationHeader = authorizationHeader, ) } + + ActionId.INTERACT_UI_ELEMENT -> { + val packageName = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME) + .valueOrNull() + + val contentDescription = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION) + .valueOrNull() + + val isFocused = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_IS_FOCUSED) + .then { Success(it.toBoolean()) }.valueOrNull() ?: false + + val text = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT).valueOrNull() + + val textSelectionStart = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_START) + .then { Success(it.toInt()) }.valueOrNull() ?: 0 + + val textSelectionEnd = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_END) + .then { Success(it.toInt()) }.valueOrNull() ?: 0 + + val isEditable = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_IS_EDITABLE) + .then { Success(it.toBoolean()) }.valueOrNull() ?: false + + val className = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_CLASS_NAME).valueOrNull() + + val viewResourceId = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID) + .valueOrNull() + + val uniqueId = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID).valueOrNull() + + val actions = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS).then { + val intList = it.split(",").map { action -> action.toInt() } + Success(intList) + }.valueOrNull() ?: emptyList() + + val nodeAction = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_NODE_ACTION).then { + Success(it.toInt()) + }.valueOrNull()!! + + ActionData.InteractUiElement( + nodeAction = nodeAction, + node = AccessibilityNodeModel( + packageName = packageName, + contentDescription = contentDescription, + isFocused = isFocused, + text = text, + textSelectionStart = textSelectionStart, + textSelectionEnd = textSelectionEnd, + isEditable = isEditable, + className = className, + uniqueId = uniqueId, + viewResourceId = viewResourceId, + actions = actions, + ), + ) + } } } @@ -538,6 +606,7 @@ object ActionDataEntityMapper { is ActionData.Text -> ActionEntity.Type.TEXT_BLOCK is ActionData.Url -> ActionEntity.Type.URL is ActionData.Sound -> ActionEntity.Type.SOUND + is ActionData.InteractUiElement -> ActionEntity.Type.INTERACT_UI_ELEMENT else -> ActionEntity.Type.SYSTEM_ACTION } @@ -579,6 +648,7 @@ object ActionDataEntityMapper { is ActionData.Text -> data.text is ActionData.Url -> data.url is ActionData.Sound -> data.soundUid + is ActionData.InteractUiElement -> "" // No data string needed for UI element interaction else -> SYSTEM_ACTION_ID_MAP[data.id]!! } @@ -750,6 +820,99 @@ object ActionDataEntityMapper { ), ) + is ActionData.InteractUiElement -> buildList { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_NODE_ACTION, + data.nodeAction.toString(), + ), + ) + + data.node.packageName?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME, + it, + ), + ) + } + + data.node.contentDescription?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION, + it, + ), + ) + } + + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_IS_FOCUSED, + data.node.isFocused.toString(), + ), + ) + + data.node.text?.let { add(EntityExtra(ActionEntity.EXTRA_ACCESSIBILITY_TEXT, it)) } + + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_START, + data.node.textSelectionStart.toString(), + ), + ) + + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_END, + data.node.textSelectionEnd.toString(), + ), + ) + + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_IS_EDITABLE, + data.node.isEditable.toString(), + ), + ) + + data.node.className?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_CLASS_NAME, + it, + ), + ) + } + + data.node.viewResourceId?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID, + it, + ), + ) + } + + data.node.uniqueId?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID, + it, + ), + ) + } + + if (data.node.actions.isNotEmpty()) { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS, + data.node.actions.joinToString(","), + ), + ) + } + } + else -> emptyList() } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt index 515474d744..011c7f825a 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt @@ -16,6 +16,7 @@ enum class ActionId { INTENT, PHONE_CALL, SOUND, + INTERACT_UI_ELEMENT, TOGGLE_WIFI, ENABLE_WIFI, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt index 755353b04d..be4ebb39e4 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt @@ -529,6 +529,8 @@ class ActionUiHelper( ActionData.DeviceControls -> getString(R.string.action_device_controls) is ActionData.HttpRequest -> action.description + + is ActionData.InteractUiElement -> getString(R.string.action_interact_ui_element_title) } fun getIcon(action: ActionData): ComposeIconInfo = when (action) { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt index e3b696fc8c..12b23dd67f 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt @@ -73,6 +73,7 @@ import io.github.sds100.keymapper.R import io.github.sds100.keymapper.system.permissions.Permission import io.github.sds100.keymapper.util.ui.compose.icons.HomeIotDevice import io.github.sds100.keymapper.util.ui.compose.icons.InstantMix +import io.github.sds100.keymapper.util.ui.compose.icons.JumpToElement import io.github.sds100.keymapper.util.ui.compose.icons.KeyMapperIcons import io.github.sds100.keymapper.util.ui.compose.icons.MatchWord import io.github.sds100.keymapper.util.ui.compose.icons.NfcOff @@ -230,6 +231,8 @@ object ActionUtils { ActionId.DISMISS_MOST_RECENT_NOTIFICATION -> ActionCategory.NOTIFICATIONS ActionId.DISMISS_ALL_NOTIFICATIONS -> ActionCategory.NOTIFICATIONS ActionId.DEVICE_CONTROLS -> ActionCategory.APPS + + ActionId.INTERACT_UI_ELEMENT -> ActionCategory.APPS } @StringRes @@ -342,6 +345,7 @@ object ActionUtils { ActionId.END_PHONE_CALL -> R.string.action_end_call ActionId.DEVICE_CONTROLS -> R.string.action_device_controls ActionId.HTTP_REQUEST -> R.string.action_http_request + ActionId.INTERACT_UI_ELEMENT -> R.string.action_interact_ui_element_title } @DrawableRes @@ -454,6 +458,7 @@ object ActionUtils { ActionId.END_PHONE_CALL -> R.drawable.ic_outline_call_end_24 ActionId.DEVICE_CONTROLS -> R.drawable.ic_home_automation ActionId.HTTP_REQUEST -> null + ActionId.INTERACT_UI_ELEMENT -> null } fun getMinApi(id: ActionId): Int = when (id) { @@ -770,6 +775,7 @@ object ActionUtils { ActionId.END_PHONE_CALL -> Icons.Outlined.CallEnd ActionId.DEVICE_CONTROLS -> KeyMapperIcons.HomeIotDevice ActionId.HTTP_REQUEST -> Icons.Outlined.Http + ActionId.INTERACT_UI_ELEMENT -> KeyMapperIcons.JumpToElement } } diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt index e99db4de73..2160feaf75 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt @@ -90,6 +90,22 @@ data class ActionEntity( const val EXTRA_HTTP_DESCRIPTION = "extra_http_description" const val EXTRA_HTTP_AUTHORIZATION_HEADER = "extra_http_authorization_header" + // Accessibility node extras + const val EXTRA_ACCESSIBILITY_PACKAGE_NAME = "extra_accessibility_package_name" + const val EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION = + "extra_accessibility_content_description" + const val EXTRA_ACCESSIBILITY_IS_FOCUSED = "extra_accessibility_is_focused" + const val EXTRA_ACCESSIBILITY_TEXT = "extra_accessibility_text" + const val EXTRA_ACCESSIBILITY_TEXT_SELECTION_START = + "extra_accessibility_text_selection_start" + const val EXTRA_ACCESSIBILITY_TEXT_SELECTION_END = "extra_accessibility_text_selection_end" + const val EXTRA_ACCESSIBILITY_IS_EDITABLE = "extra_accessibility_is_editable" + const val EXTRA_ACCESSIBILITY_CLASS_NAME = "extra_accessibility_class_name" + const val EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID = "extra_accessibility_view_resource_id" + const val EXTRA_ACCESSIBILITY_UNIQUE_ID = "extra_accessibility_unique_id" + const val EXTRA_ACCESSIBILITY_ACTIONS = "extra_accessibility_actions" + const val EXTRA_ACCESSIBILITY_NODE_ACTION = "extra_accessibility_node_action" + // DON'T CHANGE THESE. Used for JSON serialization and parsing. const val NAME_ACTION_TYPE = "type" const val NAME_DATA = "data" @@ -153,6 +169,7 @@ data class ActionEntity( INTENT, PHONE_CALL, SOUND, + INTERACT_UI_ELEMENT, } constructor( diff --git a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt index b2fa1e522f..be21fa91a6 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt @@ -14,6 +14,7 @@ interface AccessibilityNodeRepository { } class AccessibilityNodeRepositoryImpl(private val coroutineScope: CoroutineScope) : AccessibilityNodeRepository { + // TODO have a DAO to remember between app launches and so it isn't all cached in memory? override val nodes = MutableStateFlow>>(State.Data(ArrayList(128))) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt index 934c0ace2f..08f5a14e04 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt @@ -1,8 +1,13 @@ package io.github.sds100.keymapper.system.accessibility +import android.os.Build +import androidx.annotation.RequiresApi +import kotlinx.serialization.Serializable + /** * Created by sds100 on 21/04/2021. */ +@Serializable data class AccessibilityNodeModel( val packageName: String?, val contentDescription: String?, @@ -11,4 +16,14 @@ data class AccessibilityNodeModel( val textSelectionStart: Int, val textSelectionEnd: Int, val isEditable: Boolean, + val className: String?, + val viewResourceId: String?, + + @RequiresApi(Build.VERSION_CODES.TIRAMISU) + val uniqueId: String?, + + /** + * A list of the allowed accessibility node actions. + */ + val actions: List, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt index 1f16e4da93..7fff55c865 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt @@ -1,5 +1,6 @@ package io.github.sds100.keymapper.system.accessibility +import android.os.Build import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo @@ -38,6 +39,14 @@ fun AccessibilityNodeInfo.toModel(): AccessibilityNodeModel = AccessibilityNodeM textSelectionEnd = textSelectionEnd, text = text?.toString(), isEditable = isEditable, + className = className?.toString(), + uniqueId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + uniqueId + } else { + null + }, + viewResourceId = viewIdResourceName, + actions = actionList.map { it.id }, ) fun AccessibilityEvent.toModel(): AccessibilityEventModel = AccessibilityEventModel(eventTime, eventType) diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt new file mode 100644 index 0000000000..498d3bc295 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt @@ -0,0 +1,115 @@ +package io.github.sds100.keymapper.util.ui.compose.icons + +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.graphics.vector.path +import androidx.compose.ui.unit.dp + +val KeyMapperIcons.JumpToElement: ImageVector + get() { + if (_JumpToElement != null) { + return _JumpToElement!! + } + _JumpToElement = ImageVector.Builder( + name = "JumpToElement", + defaultWidth = 24.dp, + defaultHeight = 24.dp, + viewportWidth = 960f, + viewportHeight = 960f, + ).apply { + path(fill = SolidColor(Color(0xFF000000))) { + moveTo(520f, 440f) + lineTo(560f, 440f) + quadTo(577f, 440f, 588.5f, 451.5f) + quadTo(600f, 463f, 600f, 480f) + quadTo(600f, 497f, 588.5f, 508.5f) + quadTo(577f, 520f, 560f, 520f) + lineTo(480f, 520f) + quadTo(463f, 520f, 451.5f, 508.5f) + quadTo(440f, 497f, 440f, 480f) + lineTo(440f, 400f) + quadTo(440f, 383f, 451.5f, 371.5f) + quadTo(463f, 360f, 480f, 360f) + quadTo(497f, 360f, 508.5f, 371.5f) + quadTo(520f, 383f, 520f, 400f) + lineTo(520f, 440f) + close() + moveTo(800f, 440f) + lineTo(800f, 400f) + quadTo(800f, 383f, 811.5f, 371.5f) + quadTo(823f, 360f, 840f, 360f) + quadTo(857f, 360f, 868.5f, 371.5f) + quadTo(880f, 383f, 880f, 400f) + lineTo(880f, 480f) + quadTo(880f, 497f, 868.5f, 508.5f) + quadTo(857f, 520f, 840f, 520f) + lineTo(760f, 520f) + quadTo(743f, 520f, 731.5f, 508.5f) + quadTo(720f, 497f, 720f, 480f) + quadTo(720f, 463f, 731.5f, 451.5f) + quadTo(743f, 440f, 760f, 440f) + lineTo(800f, 440f) + close() + moveTo(520f, 160f) + lineTo(520f, 200f) + quadTo(520f, 217f, 508.5f, 228.5f) + quadTo(497f, 240f, 480f, 240f) + quadTo(463f, 240f, 451.5f, 228.5f) + quadTo(440f, 217f, 440f, 200f) + lineTo(440f, 120f) + quadTo(440f, 103f, 451.5f, 91.5f) + quadTo(463f, 80f, 480f, 80f) + lineTo(560f, 80f) + quadTo(577f, 80f, 588.5f, 91.5f) + quadTo(600f, 103f, 600f, 120f) + quadTo(600f, 137f, 588.5f, 148.5f) + quadTo(577f, 160f, 560f, 160f) + lineTo(520f, 160f) + close() + moveTo(800f, 160f) + lineTo(760f, 160f) + quadTo(743f, 160f, 731.5f, 148.5f) + quadTo(720f, 137f, 720f, 120f) + quadTo(720f, 103f, 731.5f, 91.5f) + quadTo(743f, 80f, 760f, 80f) + lineTo(840f, 80f) + quadTo(857f, 80f, 868.5f, 91.5f) + quadTo(880f, 103f, 880f, 120f) + lineTo(880f, 200f) + quadTo(880f, 217f, 868.5f, 228.5f) + quadTo(857f, 240f, 840f, 240f) + quadTo(823f, 240f, 811.5f, 228.5f) + quadTo(800f, 217f, 800f, 200f) + lineTo(800f, 160f) + close() + moveTo(360f, 656f) + lineTo(164f, 852f) + quadTo(153f, 863f, 136f, 863f) + quadTo(119f, 863f, 108f, 852f) + quadTo(97f, 841f, 97f, 824f) + quadTo(97f, 807f, 108f, 796f) + lineTo(304f, 600f) + lineTo(160f, 600f) + quadTo(143f, 600f, 131.5f, 588.5f) + quadTo(120f, 577f, 120f, 560f) + quadTo(120f, 543f, 131.5f, 531.5f) + quadTo(143f, 520f, 160f, 520f) + lineTo(400f, 520f) + quadTo(417f, 520f, 428.5f, 531.5f) + quadTo(440f, 543f, 440f, 560f) + lineTo(440f, 800f) + quadTo(440f, 817f, 428.5f, 828.5f) + quadTo(417f, 840f, 400f, 840f) + quadTo(383f, 840f, 371.5f, 828.5f) + quadTo(360f, 817f, 360f, 800f) + lineTo(360f, 656f) + close() + } + }.build() + + return _JumpToElement!! + } + +@Suppress("ObjectPropertyName") +private var _JumpToElement: ImageVector? = null diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 46985421d3..7d4e31a44c 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1048,6 +1048,7 @@ This action will only work if you have tapped on an input field where the keyboard is supposed to be shown. Show keyboard Hide keyboard + Show keyboard picker Switch keyboard @@ -1061,9 +1062,9 @@ Open settings Show power menu - Toggle Airplane mode - Enable Airplane mode - Disable Airplane mode + Toggle airplane mode + Enable airplane mode + Disable airplane mode Launch app Some devices require apps to have permission before they can launch apps in the background. Tap \"Read more\" to view the instructions on our website. @@ -1098,6 +1099,40 @@ Request body (optional) Authorization header (optional) You must prepend \'Bearer\' if necessary + + Interact with app element + Key Mapper can detect and interact with app elements like menus, tabs, buttons and checkboxes. You need to record yourself interacting with the app element so that Key Mapper knows what you want to do. + Start recording + Stop recording (%s min left) + Go to another app and interact with it. Key Mapper will record what you do and you can choose which interactions you want to use in your key map. Open Key Mapper again when you’re done. + + %d interaction detected + %d interactions detected + + Choose the app to interact with + Record again + Choose app element + Choose the element you want your key map to interact with. + Can\'t find what you’re looking for? + Not all apps are compatible. For incompatible apps you can try the Tap Screen action instead. + Interaction type + Select how you want to interact with the UI element. + Filter interaction type + + Tap + Tap and hold + Focus + Select + Scroll forward + Scroll backward + Expand + Collapse + + Text + + View ID + Class name + Unique ID From f1e1ec5c198b0d272a6791171ba2d03542f41b80 Mon Sep 17 00:00:00 2001 From: sds100 Date: Thu, 1 May 2025 17:30:35 +0200 Subject: [PATCH 04/21] #257 WIP: create interact with UI element screen --- .../sds100/keymapper/actions/ActionData.kt | 13 +- .../actions/ActionDataEntityMapper.kt | 84 ++------- .../keymapper/actions/CreateActionDelegate.kt | 13 ++ .../actions/PerformActionsUseCase.kt | 15 ++ .../uielement/InteractUiElementFragment.kt | 80 +++++++++ .../uielement/InteractUiElementScreen.kt | 169 ++++++++++++++++++ .../uielement/InteractUiElementUseCase.kt | 73 ++++++++ .../uielement/InteractUiElementViewModel.kt | 160 +++++++++++++++++ .../data/entities/AccessibilityNodeEntity.kt | 8 +- .../keymapper/data/entities/ActionEntity.kt | 5 - .../accessibility/AccessibilityEventModel.kt | 6 - .../AccessibilityNodeRecorder.kt | 67 ++++--- .../accessibility/AccessibilityUtils.kt | 3 - .../BaseAccessibilityServiceController.kt | 20 ++- .../accessibility/MyAccessibilityService.kt | 1 + .../RecordAccessibilityNodeState.kt | 15 ++ .../io/github/sds100/keymapper/util/Inject.kt | 14 ++ .../sds100/keymapper/util/ServiceEvent.kt | 4 + .../keymapper/util/ui/NavDestination.kt | 5 + .../keymapper/util/ui/NavigationViewModel.kt | 6 +- app/src/main/res/navigation/nav_app.xml | 23 +++ app/src/main/res/values/strings.xml | 1 + 22 files changed, 673 insertions(+), 112 deletions(-) create mode 100644 app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt delete mode 100644 app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt index 38a2587ffc..f2d4e8faa1 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt @@ -1,7 +1,6 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType -import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeModel import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.display.Orientation import io.github.sds100.keymapper.system.intents.IntentExtraModel @@ -849,8 +848,18 @@ sealed class ActionData : Comparable { @Serializable data class InteractUiElement( + val description: String, val nodeAction: Int, - val node: AccessibilityNodeModel, + val packageName: String, + val text: String?, + val contentDescription: String?, + val className: String?, + val viewResourceId: String?, + val uniqueId: String?, + /** + * A list of the allowed accessibility node actions. + */ + val nodeActions: List, ) : ActionData() { override val id: ActionId = ActionId.INTERACT_UI_ELEMENT } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index 063dd5d8c2..bcb0d4f8fc 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -5,7 +5,6 @@ import io.github.sds100.keymapper.data.db.typeconverter.ConstantTypeConverters import io.github.sds100.keymapper.data.entities.ActionEntity import io.github.sds100.keymapper.data.entities.EntityExtra import io.github.sds100.keymapper.data.entities.getData -import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeModel import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.intents.IntentExtraModel import io.github.sds100.keymapper.system.intents.IntentTarget @@ -530,29 +529,15 @@ object ActionDataEntityMapper { ActionId.INTERACT_UI_ELEMENT -> { val packageName = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME) - .valueOrNull() + .valueOrNull()!! val contentDescription = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION) .valueOrNull() - val isFocused = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_IS_FOCUSED) - .then { Success(it.toBoolean()) }.valueOrNull() ?: false - val text = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT).valueOrNull() - val textSelectionStart = - entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_START) - .then { Success(it.toInt()) }.valueOrNull() ?: 0 - - val textSelectionEnd = - entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_END) - .then { Success(it.toInt()) }.valueOrNull() ?: 0 - - val isEditable = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_IS_EDITABLE) - .then { Success(it.toBoolean()) }.valueOrNull() ?: false - val className = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_CLASS_NAME).valueOrNull() @@ -574,20 +559,15 @@ object ActionDataEntityMapper { }.valueOrNull()!! ActionData.InteractUiElement( + description = entity.data, nodeAction = nodeAction, - node = AccessibilityNodeModel( - packageName = packageName, - contentDescription = contentDescription, - isFocused = isFocused, - text = text, - textSelectionStart = textSelectionStart, - textSelectionEnd = textSelectionEnd, - isEditable = isEditable, - className = className, - uniqueId = uniqueId, - viewResourceId = viewResourceId, - actions = actions, - ), + packageName = packageName, + text = text, + contentDescription = contentDescription, + className = className, + viewResourceId = viewResourceId, + uniqueId = uniqueId, + nodeActions = actions, ) } } @@ -648,7 +628,7 @@ object ActionDataEntityMapper { is ActionData.Text -> data.text is ActionData.Url -> data.url is ActionData.Sound -> data.soundUid - is ActionData.InteractUiElement -> "" // No data string needed for UI element interaction + is ActionData.InteractUiElement -> data.description else -> SYSTEM_ACTION_ID_MAP[data.id]!! } @@ -828,7 +808,7 @@ object ActionDataEntityMapper { ), ) - data.node.packageName?.let { + data.packageName.let { add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME, @@ -837,7 +817,7 @@ object ActionDataEntityMapper { ) } - data.node.contentDescription?.let { + data.contentDescription?.let { add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION, @@ -846,37 +826,9 @@ object ActionDataEntityMapper { ) } - add( - EntityExtra( - ActionEntity.EXTRA_ACCESSIBILITY_IS_FOCUSED, - data.node.isFocused.toString(), - ), - ) - - data.node.text?.let { add(EntityExtra(ActionEntity.EXTRA_ACCESSIBILITY_TEXT, it)) } - - add( - EntityExtra( - ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_START, - data.node.textSelectionStart.toString(), - ), - ) - - add( - EntityExtra( - ActionEntity.EXTRA_ACCESSIBILITY_TEXT_SELECTION_END, - data.node.textSelectionEnd.toString(), - ), - ) - - add( - EntityExtra( - ActionEntity.EXTRA_ACCESSIBILITY_IS_EDITABLE, - data.node.isEditable.toString(), - ), - ) + data.text?.let { add(EntityExtra(ActionEntity.EXTRA_ACCESSIBILITY_TEXT, it)) } - data.node.className?.let { + data.className?.let { add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_CLASS_NAME, @@ -885,7 +837,7 @@ object ActionDataEntityMapper { ) } - data.node.viewResourceId?.let { + data.viewResourceId?.let { add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID, @@ -894,7 +846,7 @@ object ActionDataEntityMapper { ) } - data.node.uniqueId?.let { + data.uniqueId?.let { add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID, @@ -903,11 +855,11 @@ object ActionDataEntityMapper { ) } - if (data.node.actions.isNotEmpty()) { + if (data.nodeActions.isNotEmpty()) { add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS, - data.node.actions.joinToString(","), + data.nodeActions.joinToString(","), ), ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt index 7d34983a0d..ac94158e21 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt @@ -786,6 +786,19 @@ class CreateActionDelegate( } return null } + + ActionId.INTERACT_UI_ELEMENT -> { + val oldAction = if (oldData is ActionData.InteractUiElement) { + oldData + } else { + null + } + + return navigate( + "config_interact_ui_element_action", + NavDestination.InteractUiElement(oldAction), + ) + } } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt index 9734435728..3418b6edbc 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt @@ -65,6 +65,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import splitties.bitflags.withFlag @@ -799,6 +800,20 @@ class PerformActionsUseCaseImpl( authorizationHeader = action.authorizationHeader, ) } + + is ActionData.InteractUiElement -> { + if (accessibilityService.activeWindowPackage.first() != action.packageName) { + // TODO + } + + result = accessibilityService.performActionOnNode( + findNode = { node -> + // TODO compare other values + node.uniqueId == action.uniqueId + }, + performAction = { AccessibilityNodeAction(action = action.nodeAction) }, + ) + } } when (result) { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt new file mode 100644 index 0000000000..df9eb04b45 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt @@ -0,0 +1,80 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.os.Bundle +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import androidx.compose.ui.platform.ViewCompositionStrategy +import androidx.core.os.bundleOf +import androidx.fragment.app.Fragment +import androidx.fragment.app.setFragmentResult +import androidx.fragment.app.viewModels +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.withStateAtLeast +import androidx.navigation.findNavController +import androidx.navigation.fragment.findNavController +import androidx.navigation.fragment.navArgs +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.databinding.FragmentComposeBinding +import io.github.sds100.keymapper.util.Inject +import io.github.sds100.keymapper.util.launchRepeatOnLifecycle +import io.github.sds100.keymapper.util.viewLifecycleScope +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json + +class InteractUiElementFragment : Fragment() { + + companion object { + const val EXTRA_ACTION = "extra_action" + } + + private val args: InteractUiElementFragmentArgs by navArgs() + + private val viewModel by viewModels { + Inject.interactUiElementViewModel(requireContext()) + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + args.action?.let { argsAction -> viewModel.loadAction(Json.decodeFromString(argsAction)) } + + launchRepeatOnLifecycle(Lifecycle.State.CREATED) { + viewModel.returnAction.collectLatest { action -> + viewLifecycleScope.launch { + withStateAtLeast(Lifecycle.State.RESUMED) { + setFragmentResult( + args.requestKey, + bundleOf(EXTRA_ACTION to Json.encodeToString(action)), + ) + findNavController().navigateUp() + } + } + } + } + } + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle?, + ): View { + FragmentComposeBinding.inflate(inflater, container, false).apply { + composeView.apply { + // Dispose of the Composition when the view's LifecycleOwner + // is destroyed + setViewCompositionStrategy(ViewCompositionStrategy.DisposeOnViewTreeLifecycleDestroyed) + setContent { + KeyMapperTheme { + InteractUiElementScreen( + viewModel = viewModel, + navigateBack = findNavController()::navigateUp, + ) + } + } + } + return this.root + } + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt new file mode 100644 index 0000000000..aa702d54e1 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -0,0 +1,169 @@ +package io.github.sds100.keymapper.actions.uielement + +import androidx.activity.compose.BackHandler +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.calculateEndPadding +import androidx.compose.foundation.layout.calculateStartPadding +import androidx.compose.foundation.layout.displayCutoutPadding +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.rounded.ArrowBack +import androidx.compose.material.icons.rounded.Check +import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.ExtendedFloatingActionButton +import androidx.compose.material3.FloatingActionButtonDefaults +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.SnackbarHost +import androidx.compose.material3.SnackbarHostState +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.drawable +import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo + +@Composable +fun InteractUiElementScreen( + modifier: Modifier = Modifier, + viewModel: InteractUiElementViewModel, + navigateBack: () -> Unit, +) { + val snackbarHostState = SnackbarHostState() + val recordState by viewModel.recordState.collectAsStateWithLifecycle() + val selectedElementState by viewModel.selectedElementState.collectAsStateWithLifecycle() + + InteractUiElementScreen( + modifier = modifier, + recordState = recordState, + selectedElementState = selectedElementState, + onBackClick = navigateBack, + onDoneClick = { viewModel.onDoneClick() }, + snackbarHostState = snackbarHostState, + ) +} + +@Composable +private fun InteractUiElementScreen( + modifier: Modifier = Modifier, + recordState: State, + selectedElementState: SelectedUiElementState?, + onBackClick: () -> Unit = {}, + onDoneClick: () -> Unit = {}, + snackbarHostState: SnackbarHostState = SnackbarHostState(), +) { + BackHandler(onBack = onBackClick) + + Scaffold( + modifier.displayCutoutPadding(), + snackbarHost = { SnackbarHost(snackbarHostState) }, + bottomBar = { + BottomAppBar(actions = { + IconButton(onClick = onBackClick) { + Icon( + Icons.AutoMirrored.Rounded.ArrowBack, + stringResource(R.string.action_go_back), + ) + } + }, floatingActionButton = { + if (selectedElementState != null) { + ExtendedFloatingActionButton( + onClick = onDoneClick, + text = { Text(stringResource(R.string.button_done)) }, + icon = { + Icon(Icons.Rounded.Check, stringResource(R.string.button_done)) + }, + elevation = FloatingActionButtonDefaults.bottomAppBarFabElevation(), + ) + } + }) + }, + ) { innerPadding -> + val layoutDirection = LocalLayoutDirection.current + val startPadding = innerPadding.calculateStartPadding(layoutDirection) + val endPadding = innerPadding.calculateEndPadding(layoutDirection) + + Surface( + modifier = Modifier + .fillMaxSize() + .padding( + top = innerPadding.calculateTopPadding(), + bottom = innerPadding.calculateBottomPadding(), + start = startPadding, + end = endPadding, + ), + + ) { + Column { + Text( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = 16.dp, + bottom = 8.dp, + ), + text = stringResource(R.string.action_interact_ui_element_title), + style = MaterialTheme.typography.titleLarge, + ) + } + } + } +} + +@Preview +@Composable +private fun PreviewEmpty() { + KeyMapperTheme { + InteractUiElementScreen( + recordState = State.Data(RecordUiElementState.Empty), + selectedElementState = null, + ) + } +} + +@Preview +@Composable +private fun PreviewLoading() { + KeyMapperTheme { + InteractUiElementScreen( + recordState = State.Loading, + selectedElementState = null, + ) + } +} + +@Preview +@Composable +private fun PreviewSelectedElement() { + val appIcon = LocalContext.current.drawable(R.mipmap.ic_launcher_round) + + KeyMapperTheme { + InteractUiElementScreen( + recordState = State.Data(RecordUiElementState.Recorded(0)), + selectedElementState = SelectedUiElementState( + description = "Test", + appName = "Test App", + appIcon = ComposeIconInfo.Drawable(appIcon), + nodeText = "Test Node", + nodeClassName = "Test Class", + nodeViewResourceId = "io.github.sds100.keymapper:id/menu_button", + nodeUniqueId = "123", + interactionTypes = listOf(), + selectedInteraction = 0, + ), + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt new file mode 100644 index 0000000000..d6482184d4 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt @@ -0,0 +1,73 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.graphics.drawable.Drawable +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository +import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState +import io.github.sds100.keymapper.system.accessibility.ServiceAdapter +import io.github.sds100.keymapper.system.apps.PackageManagerAdapter +import io.github.sds100.keymapper.util.Result +import io.github.sds100.keymapper.util.ServiceEvent +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.mapData +import io.github.sds100.keymapper.util.onFailure +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update + +class InteractUiElementUseCaseImpl( + private val serviceAdapter: ServiceAdapter, + private val nodeRepository: AccessibilityNodeRepository, + private val packageManagerAdapter: PackageManagerAdapter, +) : InteractUiElementUseCase { + override val recordState: MutableStateFlow = + MutableStateFlow(RecordAccessibilityNodeState.Idle) + + override val interactionCount: Flow> = + nodeRepository.nodes.map { state -> state.mapData { it.size } } + + override val interactedPackages: Flow>> = nodeRepository.nodes.map { state -> + state.mapData { nodes -> + nodes.map { it.packageName }.distinct() + } + } + + override fun getInteractionsByPackage(packageName: String): Flow>> { + return nodeRepository.nodes.map { state -> + state.mapData { nodes -> + nodes.filter { it.packageName == packageName } + } + } + } + + override fun getAppName(packageName: String): Result = packageManagerAdapter.getAppName(packageName) + + override fun getAppIcon(packageName: String): Result = packageManagerAdapter.getAppIcon(packageName) + + override suspend fun startRecording(): Result<*> { + // TODO show snackbar when accessibility service is disabled error + return serviceAdapter.send(ServiceEvent.StartRecordingTrigger) + } + + override suspend fun stopRecording() { + serviceAdapter.send(ServiceEvent.StopRecordingNodes).onFailure { + recordState.update { RecordAccessibilityNodeState.Idle } + } + } +} + +interface InteractUiElementUseCase { + val recordState: StateFlow + + val interactionCount: Flow> + val interactedPackages: Flow>> + fun getInteractionsByPackage(packageName: String): Flow>> + + fun getAppName(packageName: String): Result + fun getAppIcon(packageName: String): Result + + suspend fun startRecording(): Result<*> + suspend fun stopRecording() +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt new file mode 100644 index 0000000000..b7f7493230 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -0,0 +1,160 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.view.accessibility.AccessibilityNodeInfo +import androidx.lifecycle.ViewModel +import androidx.lifecycle.ViewModelProvider +import androidx.lifecycle.viewModelScope +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.actions.ActionData +import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.dataOrNull +import io.github.sds100.keymapper.util.ui.ResourceProvider +import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch + +class InteractUiElementViewModel( + private val useCase: InteractUiElementUseCase, + private val resourceProvider: ResourceProvider, +) : ViewModel(), + ResourceProvider by resourceProvider { + + private val _returnAction: MutableSharedFlow = MutableSharedFlow() + val returnAction: SharedFlow = _returnAction + + val recordState: StateFlow> = combine( + useCase.recordState, + useCase.interactionCount, + ) { recordState, interactionCountState -> + val interactionCount = interactionCountState.dataOrNull() ?: return@combine State.Loading + + when (recordState) { + is RecordAccessibilityNodeState.CountingDown -> { + val mins = recordState.timeLeft / 60 + val secs = recordState.timeLeft % 60 + + State.Data( + RecordUiElementState.CountingDown( + timeRemaining = "$mins:$secs", + interactionCount = interactionCount, + ), + ) + } + + RecordAccessibilityNodeState.Idle -> { + if (interactionCount == 0) { + State.Data(RecordUiElementState.Empty) + } else { + State.Data(RecordUiElementState.Recorded(interactionCount = interactionCount)) + } + } + } + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + private val _selectedElementState = MutableStateFlow(null) + val selectedElementState: StateFlow = + _selectedElementState.asStateFlow() + + fun loadAction(action: ActionData.InteractUiElement) { + viewModelScope.launch { + val appName = useCase.getAppName(action.packageName).valueOrNull() ?: action.packageName + val appIcon = useCase.getAppIcon(action.packageName).valueOrNull() + ?.let { ComposeIconInfo.Drawable(it) } + + val newState = SelectedUiElementState( + description = action.description, + appName = appName, + appIcon = appIcon, + nodeText = action.text ?: action.contentDescription, + nodeClassName = action.className, + nodeViewResourceId = action.viewResourceId, + nodeUniqueId = action.uniqueId, + interactionTypes = action.nodeActions, + selectedInteraction = action.nodeAction, + ) + + _selectedElementState.update { newState } + } + } + + fun onDoneClick() { + val selectedElementState = _selectedElementState.value + if (selectedElementState == null) { + return + } + + val action = ActionData.InteractUiElement( + description = selectedElementState.description, + nodeAction = selectedElementState.selectedInteraction, + packageName = selectedElementState.appName, + text = selectedElementState.nodeText, + contentDescription = selectedElementState.nodeText, + className = selectedElementState.nodeClassName, + viewResourceId = selectedElementState.nodeViewResourceId, + uniqueId = selectedElementState.nodeUniqueId, + nodeActions = selectedElementState.interactionTypes, + ) + + _returnAction.tryEmit(action) + } + + private fun getNodeActionName(nodeAction: Int): String { + return when (nodeAction) { + AccessibilityNodeInfo.ACTION_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_click) + AccessibilityNodeInfo.ACTION_LONG_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_long_click) + AccessibilityNodeInfo.ACTION_FOCUS -> getString(R.string.action_interact_ui_element_interaction_type_focus) + AccessibilityNodeInfo.ACTION_SELECT -> getString(R.string.action_interact_ui_element_interaction_type_select) + AccessibilityNodeInfo.ACTION_SCROLL_FORWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_forward) + AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_backward) + AccessibilityNodeInfo.ACTION_EXPAND -> getString(R.string.action_interact_ui_element_interaction_type_expand) + AccessibilityNodeInfo.ACTION_COLLAPSE -> getString(R.string.action_interact_ui_element_interaction_type_collapse) + else -> getString( + R.string.action_interact_ui_element_interaction_type_unknown, + nodeAction, + ) + } + } + + @Suppress("UNCHECKED_CAST") + class Factory( + private val useCase: InteractUiElementUseCase, + private val resourceProvider: ResourceProvider, + ) : ViewModelProvider.NewInstanceFactory() { + override fun create(modelClass: Class): T { + return InteractUiElementViewModel(useCase, resourceProvider) as T + } + } +} + +data class SelectedUiElementState( + val description: String, + val appName: String, + val appIcon: ComposeIconInfo.Drawable?, + val nodeText: String?, + val nodeClassName: String?, + val nodeViewResourceId: String?, + val nodeUniqueId: String?, + val interactionTypes: List, + val selectedInteraction: Int, +) + +sealed class RecordUiElementState { + data class Recorded(val interactionCount: Int) : RecordUiElementState() + + data class CountingDown( + val timeRemaining: String, + val interactionCount: Int, + ) : RecordUiElementState() + + data object Empty : RecordUiElementState() +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt index f480ef56e6..8792942277 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -1,5 +1,11 @@ package io.github.sds100.keymapper.data.entities +import android.os.Parcelable +import kotlinx.parcelize.Parcelize +import kotlinx.serialization.Serializable + +@Serializable +@Parcelize data class AccessibilityNodeEntity( val id: Long = 0L, val parentId: Long? = null, @@ -19,4 +25,4 @@ data class AccessibilityNodeEntity( * this node. */ val userInteractedActionId: Int?, -) +) : Parcelable diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt index 2160feaf75..4385e0b3a4 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt @@ -94,12 +94,7 @@ data class ActionEntity( const val EXTRA_ACCESSIBILITY_PACKAGE_NAME = "extra_accessibility_package_name" const val EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION = "extra_accessibility_content_description" - const val EXTRA_ACCESSIBILITY_IS_FOCUSED = "extra_accessibility_is_focused" const val EXTRA_ACCESSIBILITY_TEXT = "extra_accessibility_text" - const val EXTRA_ACCESSIBILITY_TEXT_SELECTION_START = - "extra_accessibility_text_selection_start" - const val EXTRA_ACCESSIBILITY_TEXT_SELECTION_END = "extra_accessibility_text_selection_end" - const val EXTRA_ACCESSIBILITY_IS_EDITABLE = "extra_accessibility_is_editable" const val EXTRA_ACCESSIBILITY_CLASS_NAME = "extra_accessibility_class_name" const val EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID = "extra_accessibility_view_resource_id" const val EXTRA_ACCESSIBILITY_UNIQUE_ID = "extra_accessibility_unique_id" diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt deleted file mode 100644 index 3e0675d83d..0000000000 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt +++ /dev/null @@ -1,6 +0,0 @@ -package io.github.sds100.keymapper.system.accessibility - -/** - * Created by sds100 on 27/07/2021. - */ -data class AccessibilityEventModel(val eventTime: Long, val eventType: Int) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt index 71cb54bcf2..fb64958011 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -1,54 +1,68 @@ package io.github.sds100.keymapper.system.accessibility -import android.accessibilityservice.AccessibilityService import android.os.Build +import android.os.CountDownTimer import android.view.accessibility.AccessibilityEvent -import io.github.sds100.keymapper.ServiceLocator import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.update -import kotlinx.coroutines.launch class AccessibilityNodeRecorder( - private val service: AccessibilityService, - private val coroutineScope: CoroutineScope, + private val nodeRepository: AccessibilityNodeRepository, ) { companion object { private const val RECORD_DURATION = 60000L } - private val nodeRepository: AccessibilityNodeRepository by lazy { - ServiceLocator.accessibilityNodeRepository(service) - } - - private var recordJob: Job? = null - private val _isRecording = MutableStateFlow(false) - val isRecording = _isRecording.asStateFlow() + private val timerLock = Any() + private var timer: CountDownTimer? = null + private val _recordState: MutableStateFlow = + MutableStateFlow(RecordAccessibilityNodeState.Idle) + val recordState = _recordState.asStateFlow() fun startRecording() { - _isRecording.update { true } - recordJob?.cancel() - recordJob = recordJob() + synchronized(timerLock) { + timer?.cancel() + timer = object : CountDownTimer(RECORD_DURATION, 1000) { + + override fun onTick(millisUntilFinished: Long) { + _recordState.update { + RecordAccessibilityNodeState.CountingDown( + timeLeft = (millisUntilFinished / 1000).toInt(), + ) + } + } + + override fun onFinish() { + _recordState.update { RecordAccessibilityNodeState.Idle } + } + } + + timer!!.start() + } } fun stopRecording() { - recordJob?.cancel() - recordJob = null - _isRecording.update { false } + synchronized(timerLock) { + timer?.cancel() + timer = null + _recordState.update { RecordAccessibilityNodeState.Idle } + } } fun onAccessibilityEvent(event: AccessibilityEvent) { - if (!isRecording.value) { + if (_recordState.value is RecordAccessibilityNodeState.Idle) { return } val source = event.source ?: return + if (source.actionList.isNullOrEmpty()) { + return + } + val entity = AccessibilityNodeEntity( packageName = event.packageName.toString(), @@ -62,13 +76,16 @@ class AccessibilityNodeRecorder( null }, actions = source.actionList?.map { it.id } ?: emptyList(), + userInteractedActionId = event.action, ) nodeRepository.insert(entity) } - private fun recordJob() = coroutineScope.launch { - delay(RECORD_DURATION) - _isRecording.update { false } + fun teardown() { + synchronized(timerLock) { + timer?.cancel() + timer = null + } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt index 7fff55c865..773486c357 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt @@ -1,7 +1,6 @@ package io.github.sds100.keymapper.system.accessibility import android.os.Build -import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo /** @@ -48,5 +47,3 @@ fun AccessibilityNodeInfo.toModel(): AccessibilityNodeModel = AccessibilityNodeM viewResourceId = viewIdResourceName, actions = actionList.map { it.id }, ) - -fun AccessibilityEvent.toModel(): AccessibilityEventModel = AccessibilityEventModel(eventTime, eventType) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt index 7ab9aea436..bdbcc376fc 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt @@ -11,6 +11,7 @@ import io.github.sds100.keymapper.actions.PerformActionsUseCase import io.github.sds100.keymapper.constraints.DetectConstraintsUseCase import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.PreferenceDefaults +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.mappings.FingerprintGestureType import io.github.sds100.keymapper.mappings.FingerprintGesturesSupportedUseCase @@ -40,6 +41,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.drop @@ -73,6 +75,7 @@ abstract class BaseAccessibilityServiceController( private val suAdapter: SuAdapter, private val inputMethodAdapter: InputMethodAdapter, private val settingsRepository: PreferenceRepository, + private val nodeRepository: AccessibilityNodeRepository, ) { companion object { @@ -102,7 +105,7 @@ abstract class BaseAccessibilityServiceController( ) private val accessibilityNodeRecorder: AccessibilityNodeRecorder = - AccessibilityNodeRecorder(service, coroutineScope) + AccessibilityNodeRecorder(nodeRepository) private var recordingTriggerJob: Job? = null private val recordingTrigger: Boolean @@ -260,12 +263,19 @@ abstract class BaseAccessibilityServiceController( } }.launchIn(coroutineScope) + coroutineScope.launch { + accessibilityNodeRecorder.recordState.collectLatest { state -> + outputEvents.emit(ServiceEvent.OnRecordNodeStateChanged(state)) + } + } + val imeInputFocusEvents = AccessibilityEvent.TYPE_VIEW_FOCUSED or AccessibilityEvent.TYPE_VIEW_CLICKED val recordNodeEvents = AccessibilityEvent.TYPE_VIEW_FOCUSED or AccessibilityEvent.TYPE_VIEW_CLICKED or AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED + // TODO // coroutineScope.launch { // combine( // changeImeOnInputFocusFlow, @@ -320,6 +330,10 @@ abstract class BaseAccessibilityServiceController( } } + open fun onDestroy() { + accessibilityNodeRecorder.teardown() + } + open fun onConfigurationChanged(newConfig: Configuration) { } @@ -539,10 +553,9 @@ abstract class BaseAccessibilityServiceController( is ServiceEvent.TriggerKeyMap -> triggerKeyMapFromIntent(event.uid) is ServiceEvent.EnableInputMethod -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - accessibilityService.setInputMethodEnabled(event.imeId, true) + service.setInputMethodEnabled(event.imeId, true) } - is ServiceEvent.StartRecordingNodes -> { accessibilityNodeRecorder.startRecording() } @@ -556,6 +569,7 @@ abstract class BaseAccessibilityServiceController( } private fun recordTriggerJob() = coroutineScope.launch { + // TODO use Kotlin timer. repeat(RECORD_TRIGGER_TIMER_LENGTH) { iteration -> if (isActive) { val timeLeft = RECORD_TRIGGER_TIMER_LENGTH - iteration diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt index 35f9c4b55a..99205867b6 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt @@ -255,6 +255,7 @@ class MyAccessibilityService : override fun onInterrupt() {} override fun onDestroy() { + controller?.onDestroy() controller = null lifecycleRegistry.currentState = Lifecycle.State.DESTROYED diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt new file mode 100644 index 0000000000..ea1e26d091 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt @@ -0,0 +1,15 @@ +package io.github.sds100.keymapper.system.accessibility + +import kotlinx.serialization.Serializable + +@Serializable +sealed class RecordAccessibilityNodeState { + data object Idle : RecordAccessibilityNodeState() + + data class CountingDown( + /** + * The time left in seconds + */ + val timeLeft: Int, + ) : RecordAccessibilityNodeState() +} diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt index 41a4f69d47..adf8b49153 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt @@ -15,6 +15,8 @@ import io.github.sds100.keymapper.actions.sound.ChooseSoundFileUseCaseImpl import io.github.sds100.keymapper.actions.sound.ChooseSoundFileViewModel import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateViewModel import io.github.sds100.keymapper.actions.tapscreen.PickDisplayCoordinateViewModel +import io.github.sds100.keymapper.actions.uielement.InteractUiElementUseCaseImpl +import io.github.sds100.keymapper.actions.uielement.InteractUiElementViewModel import io.github.sds100.keymapper.api.KeyEventRelayServiceWrapper import io.github.sds100.keymapper.backup.BackupRestoreMappingsUseCaseImpl import io.github.sds100.keymapper.constraints.ChooseConstraintViewModel @@ -229,6 +231,7 @@ object Inject { ), inputMethodAdapter = ServiceLocator.inputMethodAdapter(service), settingsRepository = ServiceLocator.settingsRepository(service), + nodeRepository = ServiceLocator.accessibilityNodeRepository(service), ) fun chooseBluetoothDeviceViewModel(ctx: Context): ChooseBluetoothDeviceViewModel.Factory = ChooseBluetoothDeviceViewModel.Factory( @@ -248,4 +251,15 @@ object Inject { ), ServiceLocator.resourceProvider(ctx), ) + + fun interactUiElementViewModel( + ctx: Context, + ): InteractUiElementViewModel.Factory = InteractUiElementViewModel.Factory( + InteractUiElementUseCaseImpl( + serviceAdapter = ServiceLocator.accessibilityServiceAdapter(ctx), + nodeRepository = ServiceLocator.accessibilityNodeRepository(ctx), + packageManagerAdapter = ServiceLocator.packageManagerAdapter(ctx), + ), + resourceProvider = ServiceLocator.resourceProvider(ctx), + ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt b/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt index 909f82097a..4bacac2040 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.util import android.os.Parcelable import io.github.sds100.keymapper.actions.ActionData import io.github.sds100.keymapper.mappings.keymaps.trigger.KeyEventDetectionSource +import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState import io.github.sds100.keymapper.system.devices.InputDeviceInfo import kotlinx.parcelize.Parcelize import kotlinx.serialization.Serializable @@ -78,4 +79,7 @@ sealed class ServiceEvent { @Serializable data object StopRecordingNodes : ServiceEvent() + + @Serializable + data class OnRecordNodeStateChanged(val state: RecordAccessibilityNodeState) : ServiceEvent() } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt index 4e3ede84f8..de4cb1bea8 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt @@ -36,6 +36,7 @@ sealed class NavDestination { const val ID_CONFIG_KEY_MAP = "config_key_map" const val ID_SHIZUKU_SETTINGS = "shizuku_settings" const val ID_CONFIG_FLOATING_BUTTON = "config_floating_button" + const val ID_INTERACT_UI_ELEMENT_ACTION = "interact_ui_element_action" } data class ChooseApp( @@ -123,4 +124,8 @@ sealed class NavDestination { data class ConfigFloatingButton(val buttonUid: String?) : NavDestination() { override val id: String = ID_CONFIG_FLOATING_BUTTON } + + data class InteractUiElement(val action: ActionData.InteractUiElement?) : NavDestination() { + override val id: String = ID_INTERACT_UI_ELEMENT_ACTION + } } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt index eaf76d4b9d..67d57171b5 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt @@ -42,7 +42,6 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.runBlocking -import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json /** @@ -225,6 +224,11 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { is NavDestination.ConfigFloatingButton -> NavAppDirections.toConfigFloatingButton( destination.buttonUid, ) + + is NavDestination.InteractUiElement -> NavAppDirections.interactUiElement( + requestKey = requestKey, + action = destination.action?.let { Json.encodeToString(destination.action) }, + ) } fragment.findNavController().navigate(direction) diff --git a/app/src/main/res/navigation/nav_app.xml b/app/src/main/res/navigation/nav_app.xml index 4a89c3be6e..61eb7309de 100644 --- a/app/src/main/res/navigation/nav_app.xml +++ b/app/src/main/res/navigation/nav_app.xml @@ -386,4 +386,27 @@ app:argType="string" app:nullable="true" /> + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7d4e31a44c..75b2a0682f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1127,6 +1127,7 @@ Scroll backward Expand Collapse + Unknown: %d Text From 289b3c3160d14cc64378b5c8be2e44c1d59e1e0d Mon Sep 17 00:00:00 2001 From: sds100 Date: Thu, 1 May 2025 18:04:21 +0200 Subject: [PATCH 05/21] #257 start/stop recording UI elements --- .../github/sds100/keymapper/KeyMapperApp.kt | 10 ++ .../uielement/InteractUiElementFragment.kt | 7 ++ .../uielement/InteractUiElementScreen.kt | 110 +++++++++++++++++- .../uielement/InteractUiElementUseCase.kt | 23 +++- .../uielement/InteractUiElementViewModel.kt | 39 ++++++- .../io/github/sds100/keymapper/util/Inject.kt | 7 +- 6 files changed, 185 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt b/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt index b5b3275dc6..162bbd5858 100644 --- a/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt +++ b/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt @@ -12,6 +12,7 @@ import androidx.lifecycle.LifecycleObserver import androidx.lifecycle.OnLifecycleEvent import androidx.lifecycle.ProcessLifecycleOwner import androidx.multidex.MultiDexApplication +import io.github.sds100.keymapper.actions.uielement.InteractUiElementController import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.entities.LogEntryEntity import io.github.sds100.keymapper.logging.KeyMapperLoggingTree @@ -151,6 +152,15 @@ class KeyMapperApp : MultiDexApplication() { RecordTriggerController(appCoroutineScope, accessibilityServiceAdapter) } + val interactUiElementController by lazy { + InteractUiElementController( + appCoroutineScope, + accessibilityServiceAdapter, + ServiceLocator.accessibilityNodeRepository(this), + packageManagerAdapter, + ) + } + val autoGrantPermissionController by lazy { AutoGrantPermissionController( appCoroutineScope, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt index df9eb04b45..87f70c845d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt @@ -18,6 +18,7 @@ import io.github.sds100.keymapper.compose.KeyMapperTheme import io.github.sds100.keymapper.databinding.FragmentComposeBinding import io.github.sds100.keymapper.util.Inject import io.github.sds100.keymapper.util.launchRepeatOnLifecycle +import io.github.sds100.keymapper.util.ui.showPopups import io.github.sds100.keymapper.util.viewLifecycleScope import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.launch @@ -77,4 +78,10 @@ class InteractUiElementFragment : Fragment() { return this.root } } + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + super.onViewCreated(view, savedInstanceState) + + viewModel.showPopups(this, view) + } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index aa702d54e1..616518e520 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -1,21 +1,28 @@ package io.github.sds100.keymapper.actions.uielement import androidx.activity.compose.BackHandler +import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.calculateEndPadding import androidx.compose.foundation.layout.calculateStartPadding import androidx.compose.foundation.layout.displayCutoutPadding import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.rounded.ArrowBack import androidx.compose.material.icons.rounded.Check import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.ExtendedFloatingActionButton +import androidx.compose.material3.FilledTonalButton import androidx.compose.material3.FloatingActionButtonDefaults import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton import androidx.compose.material3.Scaffold import androidx.compose.material3.SnackbarHost import androidx.compose.material3.SnackbarHostState @@ -27,11 +34,13 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalLayoutDirection import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import io.github.sds100.keymapper.R import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.compose.LocalCustomColorsPalette import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.drawable import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo @@ -51,7 +60,8 @@ fun InteractUiElementScreen( recordState = recordState, selectedElementState = selectedElementState, onBackClick = navigateBack, - onDoneClick = { viewModel.onDoneClick() }, + onDoneClick = viewModel::onDoneClick, + onRecordClick = viewModel::onRecordClick, snackbarHostState = snackbarHostState, ) } @@ -63,6 +73,7 @@ private fun InteractUiElementScreen( selectedElementState: SelectedUiElementState?, onBackClick: () -> Unit = {}, onDoneClick: () -> Unit = {}, + onRecordClick: () -> Unit = {}, snackbarHostState: SnackbarHostState = SnackbarHostState(), ) { BackHandler(onBack = onBackClick) @@ -118,11 +129,108 @@ private fun InteractUiElementScreen( text = stringResource(R.string.action_interact_ui_element_title), style = MaterialTheme.typography.titleLarge, ) + + Text( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + text = stringResource(R.string.action_interact_ui_element_description), + style = MaterialTheme.typography.bodyMedium, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + RecordingSection( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + state = recordState, + onRecordClick = onRecordClick, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + if (selectedElementState != null) { + SelectedElementSection(modifier = Modifier.fillMaxWidth(), selectedElementState) + } } } } } +@Composable +private fun RecordingSection( + modifier: Modifier = Modifier, + state: State, + onRecordClick: () -> Unit = {}, +) { + Column(modifier = modifier) { + when (state) { + is State.Data -> { + RecordButton( + modifier = Modifier.fillMaxWidth(), + state = state.data, + onClick = onRecordClick, + ) + } + + State.Loading -> TODO() + } + } +} + +@Composable +private fun RecordButton( + modifier: Modifier, + state: RecordUiElementState, + onClick: () -> Unit, +) { + val text: String = when (state) { + is RecordUiElementState.Empty -> stringResource(R.string.action_interact_ui_element_start_recording) + is RecordUiElementState.Recorded -> stringResource(R.string.action_interact_ui_element_record_again) + is RecordUiElementState.CountingDown -> stringResource( + R.string.action_interact_ui_element_stop_recording, + state.timeRemaining, + ) + } + + if (state is RecordUiElementState.Recorded) { + OutlinedButton( + modifier = modifier, + onClick = onClick, + colors = ButtonDefaults.outlinedButtonColors().copy( + contentColor = LocalCustomColorsPalette.current.red, + ), + border = BorderStroke(1.dp, color = LocalCustomColorsPalette.current.red), + ) { + Text( + text = text, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } else { + FilledTonalButton( + modifier = modifier, + onClick = onClick, + colors = ButtonDefaults.filledTonalButtonColors().copy( + containerColor = LocalCustomColorsPalette.current.red, + contentColor = LocalCustomColorsPalette.current.onRed, + ), + ) { + Text( + text = text, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } +} + +@Composable +private fun SelectedElementSection(modifier: Modifier = Modifier, state: SelectedUiElementState) { +} + @Preview @Composable private fun PreviewEmpty() { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt index d6482184d4..79688b1629 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt @@ -11,13 +11,18 @@ import io.github.sds100.keymapper.util.ServiceEvent import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.mapData import io.github.sds100.keymapper.util.onFailure +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.update -class InteractUiElementUseCaseImpl( +class InteractUiElementController( + private val coroutineScope: CoroutineScope, private val serviceAdapter: ServiceAdapter, private val nodeRepository: AccessibilityNodeRepository, private val packageManagerAdapter: PackageManagerAdapter, @@ -34,6 +39,13 @@ class InteractUiElementUseCaseImpl( } } + init { + serviceAdapter.eventReceiver + .filterIsInstance() + .onEach { event -> recordState.update { event.state } } + .launchIn(coroutineScope) + } + override fun getInteractionsByPackage(packageName: String): Flow>> { return nodeRepository.nodes.map { state -> state.mapData { nodes -> @@ -47,8 +59,7 @@ class InteractUiElementUseCaseImpl( override fun getAppIcon(packageName: String): Result = packageManagerAdapter.getAppIcon(packageName) override suspend fun startRecording(): Result<*> { - // TODO show snackbar when accessibility service is disabled error - return serviceAdapter.send(ServiceEvent.StartRecordingTrigger) + return serviceAdapter.send(ServiceEvent.StartRecordingNodes) } override suspend fun stopRecording() { @@ -56,6 +67,10 @@ class InteractUiElementUseCaseImpl( recordState.update { RecordAccessibilityNodeState.Idle } } } + + override fun startService(): Boolean { + return serviceAdapter.start() + } } interface InteractUiElementUseCase { @@ -70,4 +85,6 @@ interface InteractUiElementUseCase { suspend fun startRecording(): Result<*> suspend fun stopRecording() + + fun startService(): Boolean } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt index b7f7493230..014731b87b 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -7,9 +7,15 @@ import androidx.lifecycle.viewModelScope import io.github.sds100.keymapper.R import io.github.sds100.keymapper.actions.ActionData import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState +import io.github.sds100.keymapper.util.Error import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.dataOrNull +import io.github.sds100.keymapper.util.ifIsData +import io.github.sds100.keymapper.util.onFailure +import io.github.sds100.keymapper.util.ui.PopupViewModel +import io.github.sds100.keymapper.util.ui.PopupViewModelImpl import io.github.sds100.keymapper.util.ui.ResourceProvider +import io.github.sds100.keymapper.util.ui.ViewModelHelper import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo import io.github.sds100.keymapper.util.valueOrNull import kotlinx.coroutines.flow.MutableSharedFlow @@ -27,7 +33,8 @@ class InteractUiElementViewModel( private val useCase: InteractUiElementUseCase, private val resourceProvider: ResourceProvider, ) : ViewModel(), - ResourceProvider by resourceProvider { + ResourceProvider by resourceProvider, + PopupViewModel by PopupViewModelImpl() { private val _returnAction: MutableSharedFlow = MutableSharedFlow() val returnAction: SharedFlow = _returnAction @@ -108,6 +115,36 @@ class InteractUiElementViewModel( _returnAction.tryEmit(action) } + fun onRecordClick() { + recordState.value.ifIsData { recordState -> + viewModelScope.launch { + when (recordState) { + is RecordUiElementState.CountingDown -> useCase.stopRecording() + RecordUiElementState.Empty -> startRecording() + is RecordUiElementState.Recorded -> startRecording() + } + } + } + } + + private suspend fun startRecording() { + useCase.startRecording().onFailure { error -> + if (error == Error.AccessibilityServiceDisabled) { + ViewModelHelper.handleAccessibilityServiceStoppedDialog( + this, + this, + startService = { useCase.startService() }, + ) + } else if (error == Error.AccessibilityServiceCrashed) { + ViewModelHelper.handleAccessibilityServiceCrashedDialog( + this, + this, + restartService = { useCase.startService() }, + ) + } + } + } + private fun getNodeActionName(nodeAction: Int): String { return when (nodeAction) { AccessibilityNodeInfo.ACTION_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_click) diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt index adf8b49153..6bb6786364 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt @@ -15,7 +15,6 @@ import io.github.sds100.keymapper.actions.sound.ChooseSoundFileUseCaseImpl import io.github.sds100.keymapper.actions.sound.ChooseSoundFileViewModel import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateViewModel import io.github.sds100.keymapper.actions.tapscreen.PickDisplayCoordinateViewModel -import io.github.sds100.keymapper.actions.uielement.InteractUiElementUseCaseImpl import io.github.sds100.keymapper.actions.uielement.InteractUiElementViewModel import io.github.sds100.keymapper.api.KeyEventRelayServiceWrapper import io.github.sds100.keymapper.backup.BackupRestoreMappingsUseCaseImpl @@ -255,11 +254,7 @@ object Inject { fun interactUiElementViewModel( ctx: Context, ): InteractUiElementViewModel.Factory = InteractUiElementViewModel.Factory( - InteractUiElementUseCaseImpl( - serviceAdapter = ServiceLocator.accessibilityServiceAdapter(ctx), - nodeRepository = ServiceLocator.accessibilityNodeRepository(ctx), - packageManagerAdapter = ServiceLocator.packageManagerAdapter(ctx), - ), + (ctx.applicationContext as KeyMapperApp).interactUiElementController, resourceProvider = ServiceLocator.resourceProvider(ctx), ) } From a55808ac92b3f19877b5366c0fb1777f631d30fd Mon Sep 17 00:00:00 2001 From: sds100 Date: Sat, 3 May 2025 15:23:53 +0200 Subject: [PATCH 06/21] feat: rename tap screen actions inside key maps --- CHANGELOG.md | 4 ++++ .../java/io/github/sds100/keymapper/actions/ActionUiHelper.kt | 2 +- app/src/main/res/values/strings.xml | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b43105a25..7e501ffdd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - #699 Time constraints ⏰ +## Changed + +- Rename tap screen actions inside key maps. + ## [3.0.1](https://github.com/sds100/KeyMapper/releases/tag/v3.0.1) #### 28 April 2025 diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt index be4ebb39e4..e782916e4b 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt @@ -356,7 +356,7 @@ class ActionUiHelper( } else { getString( R.string.description_tap_coordinate_with_description, - arrayOf(action.x, action.y, action.description), + action.description, ) } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 75b2a0682f..beb6c3c235 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -98,8 +98,8 @@ Input %s through shell Input %s%s from %s Open %s - Tap coordinates %d, %d - Tap coordinates %d, %d (%s) + Tap screen (%d, %d) + Tap screen (%s) Swipe with %d finger(s) from coordinates %d/%d to %d/%d in %dms Swipe with %d finger(s) from coordinates %d/%d to %d/%d in %dms (%s) %s with %d finger(s) on coordinates %d/%d with a pinch distance of %dpx in %dms From 21d00621190118d4878e487a44ec8c9a5650a5c6 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sat, 3 May 2025 16:55:20 +0200 Subject: [PATCH 07/21] #257 feat: show teh number of interactions --- .../uielement/InteractUiElementScreen.kt | 130 ++++++++++++++++-- .../util/ui/compose/icons/AdGroup.kt | 78 +++++++++++ 2 files changed, 193 insertions(+), 15 deletions(-) create mode 100644 app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index 616518e520..91d1e88d60 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.actions.uielement import androidx.activity.compose.BackHandler import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.calculateEndPadding import androidx.compose.foundation.layout.calculateStartPadding @@ -11,9 +12,11 @@ import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.rounded.ArrowBack import androidx.compose.material.icons.rounded.Check +import androidx.compose.material.icons.rounded.ChevronRight import androidx.compose.material3.BottomAppBar import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.ExtendedFloatingActionButton @@ -21,6 +24,8 @@ import androidx.compose.material3.FilledTonalButton import androidx.compose.material3.FloatingActionButtonDefaults import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.LocalContentColor import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.Scaffold @@ -28,11 +33,15 @@ import androidx.compose.material3.SnackbarHost import androidx.compose.material3.SnackbarHostState import androidx.compose.material3.Surface import androidx.compose.material3.Text +import androidx.compose.material3.contentColorFor import androidx.compose.runtime.Composable +import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.pluralStringResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.tooling.preview.Preview @@ -44,6 +53,8 @@ import io.github.sds100.keymapper.compose.LocalCustomColorsPalette import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.drawable import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.icons.AdGroup +import io.github.sds100.keymapper.util.ui.compose.icons.KeyMapperIcons @Composable fun InteractUiElementScreen( @@ -90,14 +101,34 @@ private fun InteractUiElementScreen( ) } }, floatingActionButton = { - if (selectedElementState != null) { + val isEnabled = selectedElementState != null + val containerColor = if (isEnabled) { + FloatingActionButtonDefaults.containerColor + } else { + FloatingActionButtonDefaults.containerColor.copy(alpha = 0.5f) + } + + val contentColor = if (isEnabled) { + MaterialTheme.colorScheme.contentColorFor(containerColor) + } else { + MaterialTheme.colorScheme.contentColorFor(containerColor).copy(alpha = 0.5f) + } + + CompositionLocalProvider( + LocalContentColor provides contentColor, + ) { ExtendedFloatingActionButton( - onClick = onDoneClick, + onClick = if (isEnabled) { + onDoneClick + } else { + {} + }, text = { Text(stringResource(R.string.button_done)) }, icon = { Icon(Icons.Rounded.Check, stringResource(R.string.button_done)) }, elevation = FloatingActionButtonDefaults.bottomAppBarFabElevation(), + containerColor = containerColor, ) } }) @@ -167,6 +198,20 @@ private fun RecordingSection( Column(modifier = modifier) { when (state) { is State.Data -> { + val interactionCount: Int = when (state.data) { + is RecordUiElementState.CountingDown -> state.data.interactionCount + is RecordUiElementState.Recorded -> state.data.interactionCount + RecordUiElementState.Empty -> 0 + } + + InteractionCountBox( + modifier = Modifier.fillMaxWidth(), + interactionCount = interactionCount, + onClick = {}, + ) + + Spacer(modifier = Modifier.height(8.dp)) + RecordButton( modifier = Modifier.fillMaxWidth(), state = state.data, @@ -174,7 +219,60 @@ private fun RecordingSection( ) } - State.Loading -> TODO() + State.Loading -> { + Spacer(modifier = Modifier.height(16.dp)) + LinearProgressIndicator(modifier = Modifier.fillMaxWidth()) + Spacer(modifier = Modifier.height(16.dp)) + } + } + } +} + +@Composable +private fun InteractionCountBox( + modifier: Modifier = Modifier, + interactionCount: Int, + onClick: () -> Unit, +) { + val enabled = interactionCount > 0 + + val color = if (enabled) { + MaterialTheme.colorScheme.onSurface + } else { + MaterialTheme.colorScheme.onSurface.copy(alpha = 0.5f) + } + + Surface(modifier = modifier, onClick = onClick, enabled = enabled) { + CompositionLocalProvider( + LocalContentColor provides color, + ) { + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + ) { + Spacer(modifier = Modifier.width(16.dp)) + Icon(imageVector = KeyMapperIcons.AdGroup, contentDescription = null) + Spacer(modifier = Modifier.width(16.dp)) + Column(modifier = Modifier.weight(1f)) { + Text( + pluralStringResource( + R.plurals.action_interact_ui_element_interactions_detected, + interactionCount, + interactionCount, + ), + style = MaterialTheme.typography.bodyLarge, + ) + + Text( + stringResource(R.string.action_interact_ui_element_choose_interaction), + style = MaterialTheme.typography.bodyMedium, + ) + } + + Spacer(modifier = Modifier.width(16.dp)) + Icon(imageVector = Icons.Rounded.ChevronRight, contentDescription = null) + Spacer(modifier = Modifier.width(16.dp)) + } } } } @@ -229,6 +327,8 @@ private fun RecordButton( @Composable private fun SelectedElementSection(modifier: Modifier = Modifier, state: SelectedUiElementState) { + Column(modifier = modifier) { + } } @Preview @@ -242,17 +342,6 @@ private fun PreviewEmpty() { } } -@Preview -@Composable -private fun PreviewLoading() { - KeyMapperTheme { - InteractUiElementScreen( - recordState = State.Loading, - selectedElementState = null, - ) - } -} - @Preview @Composable private fun PreviewSelectedElement() { @@ -260,7 +349,7 @@ private fun PreviewSelectedElement() { KeyMapperTheme { InteractUiElementScreen( - recordState = State.Data(RecordUiElementState.Recorded(0)), + recordState = State.Data(RecordUiElementState.Recorded(3)), selectedElementState = SelectedUiElementState( description = "Test", appName = "Test App", @@ -275,3 +364,14 @@ private fun PreviewSelectedElement() { ) } } + +@Preview +@Composable +private fun PreviewLoading() { + KeyMapperTheme { + InteractUiElementScreen( + recordState = State.Loading, + selectedElementState = null, + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt new file mode 100644 index 0000000000..05f0f4db8b --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt @@ -0,0 +1,78 @@ +package io.github.sds100.keymapper.util.ui.compose.icons + +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.graphics.vector.path +import androidx.compose.ui.unit.dp + +val KeyMapperIcons.AdGroup: ImageVector + get() { + if (_AdGroup != null) { + return _AdGroup!! + } + _AdGroup = ImageVector.Builder( + name = "AdGroup", + defaultWidth = 24.dp, + defaultHeight = 24.dp, + viewportWidth = 960f, + viewportHeight = 960f, + ).apply { + path(fill = SolidColor(Color(0xFF000000))) { + moveTo(320f, 640f) + lineTo(800f, 640f) + quadTo(800f, 640f, 800f, 640f) + quadTo(800f, 640f, 800f, 640f) + lineTo(800f, 240f) + lineTo(320f, 240f) + lineTo(320f, 640f) + quadTo(320f, 640f, 320f, 640f) + quadTo(320f, 640f, 320f, 640f) + close() + moveTo(320f, 720f) + quadTo(287f, 720f, 263.5f, 696.5f) + quadTo(240f, 673f, 240f, 640f) + lineTo(240f, 160f) + quadTo(240f, 127f, 263.5f, 103.5f) + quadTo(287f, 80f, 320f, 80f) + lineTo(800f, 80f) + quadTo(833f, 80f, 856.5f, 103.5f) + quadTo(880f, 127f, 880f, 160f) + lineTo(880f, 640f) + quadTo(880f, 673f, 856.5f, 696.5f) + quadTo(833f, 720f, 800f, 720f) + lineTo(320f, 720f) + close() + moveTo(160f, 880f) + quadTo(127f, 880f, 103.5f, 856.5f) + quadTo(80f, 833f, 80f, 800f) + lineTo(80f, 240f) + lineTo(160f, 240f) + lineTo(160f, 800f) + quadTo(160f, 800f, 160f, 800f) + quadTo(160f, 800f, 160f, 800f) + lineTo(720f, 800f) + lineTo(720f, 880f) + lineTo(160f, 880f) + close() + moveTo(320f, 160f) + quadTo(320f, 160f, 320f, 160f) + quadTo(320f, 160f, 320f, 160f) + lineTo(320f, 640f) + quadTo(320f, 640f, 320f, 640f) + quadTo(320f, 640f, 320f, 640f) + lineTo(320f, 640f) + quadTo(320f, 640f, 320f, 640f) + quadTo(320f, 640f, 320f, 640f) + lineTo(320f, 160f) + quadTo(320f, 160f, 320f, 160f) + quadTo(320f, 160f, 320f, 160f) + close() + } + }.build() + + return _AdGroup!! + } + +@Suppress("ObjectPropertyName") +private var _AdGroup: ImageVector? = null From 22c3226689158a8f2582f5cd427f6cb3d6f7a718 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sat, 3 May 2025 17:33:57 +0200 Subject: [PATCH 08/21] #257 recording nodes works --- .../uielement/InteractUiElementScreen.kt | 38 ++++-------- .../AccessibilityNodeRecorder.kt | 2 +- .../BaseAccessibilityServiceController.kt | 62 ++++++++++--------- 3 files changed, 45 insertions(+), 57 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index 91d1e88d60..72b405b0ef 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -33,7 +33,6 @@ import androidx.compose.material3.SnackbarHost import androidx.compose.material3.SnackbarHostState import androidx.compose.material3.Surface import androidx.compose.material3.Text -import androidx.compose.material3.contentColorFor import androidx.compose.runtime.Composable import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.runtime.getValue @@ -101,34 +100,14 @@ private fun InteractUiElementScreen( ) } }, floatingActionButton = { - val isEnabled = selectedElementState != null - val containerColor = if (isEnabled) { - FloatingActionButtonDefaults.containerColor - } else { - FloatingActionButtonDefaults.containerColor.copy(alpha = 0.5f) - } - - val contentColor = if (isEnabled) { - MaterialTheme.colorScheme.contentColorFor(containerColor) - } else { - MaterialTheme.colorScheme.contentColorFor(containerColor).copy(alpha = 0.5f) - } - - CompositionLocalProvider( - LocalContentColor provides contentColor, - ) { + if (selectedElementState != null) { ExtendedFloatingActionButton( - onClick = if (isEnabled) { - onDoneClick - } else { - {} - }, + onClick = onDoneClick, text = { Text(stringResource(R.string.button_done)) }, icon = { Icon(Icons.Rounded.Check, stringResource(R.string.button_done)) }, elevation = FloatingActionButtonDefaults.bottomAppBarFabElevation(), - containerColor = containerColor, ) } }) @@ -242,15 +221,21 @@ private fun InteractionCountBox( MaterialTheme.colorScheme.onSurface.copy(alpha = 0.5f) } - Surface(modifier = modifier, onClick = onClick, enabled = enabled) { + Surface( + modifier = modifier, + onClick = onClick, + enabled = enabled, + shape = MaterialTheme.shapes.medium, + ) { CompositionLocalProvider( LocalContentColor provides color, ) { Row( - modifier = Modifier.fillMaxWidth(), + modifier = Modifier + .fillMaxWidth() + .padding(8.dp), verticalAlignment = Alignment.CenterVertically, ) { - Spacer(modifier = Modifier.width(16.dp)) Icon(imageVector = KeyMapperIcons.AdGroup, contentDescription = null) Spacer(modifier = Modifier.width(16.dp)) Column(modifier = Modifier.weight(1f)) { @@ -271,7 +256,6 @@ private fun InteractionCountBox( Spacer(modifier = Modifier.width(16.dp)) Icon(imageVector = Icons.Rounded.ChevronRight, contentDescription = null) - Spacer(modifier = Modifier.width(16.dp)) } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt index fb64958011..d031696e04 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -66,7 +66,7 @@ class AccessibilityNodeRecorder( val entity = AccessibilityNodeEntity( packageName = event.packageName.toString(), - text = source.text.firstOrNull()?.toString(), + text = source.text?.toString(), contentDescription = source.contentDescription?.toString(), className = source.className?.toString(), viewResourceId = source.viewIdResourceName, diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt index bdbcc376fc..d2206d813e 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt @@ -41,6 +41,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged @@ -49,6 +50,7 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.flow.update import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -271,37 +273,39 @@ abstract class BaseAccessibilityServiceController( val imeInputFocusEvents = AccessibilityEvent.TYPE_VIEW_FOCUSED or AccessibilityEvent.TYPE_VIEW_CLICKED + val recordNodeEvents = AccessibilityEvent.TYPE_VIEW_FOCUSED or AccessibilityEvent.TYPE_VIEW_CLICKED or - AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED - - // TODO -// coroutineScope.launch { -// combine( -// changeImeOnInputFocusFlow, -// accessibilityNodeRecorder.isRecording -// ) { changeImeOnInputFocus, isRecordingNodes -> -// serviceEventTypes.update { eventTypes -> -// var newEventTypes = eventTypes -// -// -// -// newEventTypes -// } -// -// }.collect() -// } -// changeImeOnInputFocusFlow.onEach { changeImeOnInputFocus -> -// if (changeImeOnInputFocus) { -// serviceEventTypes.value = serviceEventTypes.value -// .withFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) -// .withFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) -// } else { -// serviceEventTypes.value = serviceEventTypes.value -// .minusFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) -// .minusFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) -// } -// }.launchIn(coroutineScope) + AccessibilityEvent.TYPE_VIEW_LONG_CLICKED or + AccessibilityEvent.TYPE_VIEW_SELECTED or + AccessibilityEvent.TYPE_VIEW_SCROLLED + + coroutineScope.launch { + combine( + changeImeOnInputFocusFlow, + accessibilityNodeRecorder.recordState, + ) { changeImeOnInputFocus, recordState -> + + serviceEventTypes.update { eventTypes -> + var newEventTypes = eventTypes + + if (!changeImeOnInputFocus && recordState == RecordAccessibilityNodeState.Idle) { + newEventTypes = + newEventTypes and (imeInputFocusEvents or recordNodeEvents).inv() + } else { + if (changeImeOnInputFocus) { + newEventTypes = newEventTypes or imeInputFocusEvents + } + + if (recordState is RecordAccessibilityNodeState.CountingDown) { + newEventTypes = newEventTypes or recordNodeEvents + } + } + + newEventTypes + } + }.collect() + } } open fun onServiceConnected() { From bb6ccba697b555f945cf2acd29f24031e271692d Mon Sep 17 00:00:00 2001 From: sds100 Date: Sat, 3 May 2025 21:44:36 +0200 Subject: [PATCH 09/21] #257 list recorded apps --- app/build.gradle | 10 +- .../keymapper/actions/ChooseActionScreen.kt | 86 ++----- .../uielement/InteractUiElementFragment.kt | 3 + .../uielement/InteractUiElementScreen.kt | 85 +++++-- .../uielement/InteractUiElementViewModel.kt | 43 ++++ .../keymapper/system/apps/ChooseAppScreen.kt | 216 ++++++++++++++++++ .../util/ui/compose/SearchAppBarActions.kt | 90 ++++++++ .../util/ui/compose/SimpleListItem.kt | 107 ++++++--- app/src/main/res/values/strings.xml | 2 +- 9 files changed, 513 insertions(+), 129 deletions(-) create mode 100644 app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt diff --git a/app/build.gradle b/app/build.gradle index c7ed7dc671..e442585f81 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -165,7 +165,7 @@ dependencies { compileOnly project(":systemstubs") - def room_version = "2.6.1" + def room_version = "2.7.1" def coroutinesVersion = "1.9.0" def nav_version = '2.8.9' def epoxy_version = "4.6.2" @@ -179,7 +179,7 @@ dependencies { implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.8.0" // random stuff - implementation "com.google.android.material:material:1.13.0-alpha12" + implementation "com.google.android.material:material:1.13.0-alpha13" implementation "com.github.salomonbrys.kotson:kotson:2.5.0" implementation "com.airbnb.android:epoxy:$epoxy_version" implementation "com.airbnb.android:epoxy-databinding:$epoxy_version" @@ -192,7 +192,7 @@ dependencies { implementation "dev.rikka.shizuku:api:$shizuku_version" implementation "dev.rikka.shizuku:provider:$shizuku_version" implementation "org.lsposed.hiddenapibypass:hiddenapibypass:4.3" - proImplementation 'com.revenuecat.purchases:purchases:8.15.0' + proImplementation 'com.revenuecat.purchases:purchases:8.17.0' proImplementation "com.airbnb.android:lottie-compose:6.6.3" implementation("com.squareup.okhttp3:okhttp:4.12.0") coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") @@ -207,7 +207,7 @@ dependencies { // androidx implementation "androidx.legacy:legacy-support-core-ui:1.0.0" - implementation "androidx.core:core-ktx:1.15.0" + implementation "androidx.core:core-ktx:1.16.0" implementation "androidx.activity:activity-ktx:1.10.1" implementation "androidx.fragment:fragment-ktx:1.8.6" @@ -233,7 +233,7 @@ dependencies { ksp "androidx.room:room-compiler:$room_version" // Compose - Dependency composeBom = platform('androidx.compose:compose-bom-beta:2025.03.01') + Dependency composeBom = platform('androidx.compose:compose-bom-beta:2025.04.01') implementation composeBom implementation 'androidx.compose.foundation:foundation' implementation "androidx.compose.ui:ui-android" diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt index 9001bf183b..34cc3a8616 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt @@ -15,27 +15,18 @@ import androidx.compose.foundation.lazy.grid.GridCells import androidx.compose.foundation.lazy.grid.LazyVerticalGrid import androidx.compose.foundation.lazy.grid.items import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.automirrored.rounded.ArrowBack import androidx.compose.material.icons.rounded.Android import androidx.compose.material.icons.rounded.Bluetooth -import androidx.compose.material.icons.rounded.Search import androidx.compose.material.icons.rounded.Wifi import androidx.compose.material3.BottomAppBar import androidx.compose.material3.CircularProgressIndicator -import androidx.compose.material3.DockedSearchBar import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold -import androidx.compose.material3.SearchBarDefaults import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.saveable.rememberSaveable -import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalLayoutDirection @@ -51,7 +42,8 @@ import io.github.sds100.keymapper.R import io.github.sds100.keymapper.compose.KeyMapperTheme import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo -import io.github.sds100.keymapper.util.ui.compose.SimpleListItem +import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions +import io.github.sds100.keymapper.util.ui.compose.SimpleListItemFixedHeight import io.github.sds100.keymapper.util.ui.compose.SimpleListItemGroup import io.github.sds100.keymapper.util.ui.compose.SimpleListItemHeader import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel @@ -92,69 +84,18 @@ private fun ChooseActionScreen( onClickAction: (String) -> Unit = {}, onNavigateBack: () -> Unit = {}, ) { - var isExpanded: Boolean by rememberSaveable { mutableStateOf(false) } - Scaffold( modifier = modifier.displayCutoutPadding(), bottomBar = { BottomAppBar( modifier = Modifier.imePadding(), actions = { - IconButton(onClick = { - if (isExpanded) { - onCloseSearch() - isExpanded = false - } else { - onNavigateBack() - } - }) { - Icon( - Icons.AutoMirrored.Rounded.ArrowBack, - contentDescription = stringResource(R.string.bottom_app_bar_back_content_description), - ) - } - - DockedSearchBar( - modifier = Modifier.align(Alignment.CenterVertically), - inputField = { - SearchBarDefaults.InputField( - modifier = Modifier.align(Alignment.CenterVertically), - onSearch = { - onQueryChange(it) - isExpanded = false - }, - leadingIcon = { - Icon( - Icons.Rounded.Search, - contentDescription = null, - ) - }, - enabled = state is State.Data, - placeholder = { Text(stringResource(R.string.search_placeholder)) }, - query = query ?: "", - onQueryChange = onQueryChange, - expanded = isExpanded, - onExpandedChange = { expanded -> - if (expanded) { - isExpanded = true - } else { - onCloseSearch() - isExpanded = false - } - }, - ) - }, - // This is false to prevent an empty "content" showing underneath. - expanded = isExpanded, - onExpandedChange = { expanded -> - if (expanded) { - isExpanded = true - } else { - onCloseSearch() - isExpanded = false - } - }, - content = {}, + SearchAppBarActions( + onCloseSearch = onCloseSearch, + onNavigateBack = onNavigateBack, + onQueryChange = onQueryChange, + enabled = state is State.Data, + query = query, ) }, ) @@ -260,7 +201,7 @@ private fun ListScreen( group.items, contentType = { "list_item" }, ) { model -> - SimpleListItem( + SimpleListItemFixedHeight( modifier = Modifier.fillMaxWidth(), model = model, onClick = { onClickAction(model.id) }, @@ -363,6 +304,15 @@ private fun PreviewGrid() { isEnabled = false, ), + SimpleListItemModel( + "long", + title = "Very very very very very very very long title", + icon = ComposeIconInfo.Vector(Icons.Rounded.Bluetooth), + subtitle = null, + isSubtitleError = true, + isEnabled = false, + ), + ), ), diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt index 87f70c845d..ddde228691 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt @@ -4,6 +4,8 @@ import android.os.Bundle import android.view.LayoutInflater import android.view.View import android.view.ViewGroup +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.ui.Modifier import androidx.compose.ui.platform.ViewCompositionStrategy import androidx.core.os.bundleOf import androidx.fragment.app.Fragment @@ -69,6 +71,7 @@ class InteractUiElementFragment : Fragment() { setContent { KeyMapperTheme { InteractUiElementScreen( + modifier = Modifier.fillMaxSize(), viewModel = viewModel, navigateBack = findNavController()::navigateUp, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index 72b405b0ef..b6c94c3943 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -1,6 +1,7 @@ package io.github.sds100.keymapper.actions.uielement import androidx.activity.compose.BackHandler +import androidx.compose.animation.AnimatedContentTransitionScope import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row @@ -46,14 +47,23 @@ import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.navigation.compose.NavHost +import androidx.navigation.compose.composable +import androidx.navigation.compose.rememberNavController import io.github.sds100.keymapper.R import io.github.sds100.keymapper.compose.KeyMapperTheme import io.github.sds100.keymapper.compose.LocalCustomColorsPalette +import io.github.sds100.keymapper.system.apps.ChooseAppScreen import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.drawable import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo import io.github.sds100.keymapper.util.ui.compose.icons.AdGroup import io.github.sds100.keymapper.util.ui.compose.icons.KeyMapperIcons +import kotlinx.coroutines.flow.update + +private const val DEST_LANDING = "landing" +private const val DEST_SELECT_APP = "select_app" +private const val DEST_SELECT_ELEMENT = "select_element" @Composable fun InteractUiElementScreen( @@ -61,32 +71,70 @@ fun InteractUiElementScreen( viewModel: InteractUiElementViewModel, navigateBack: () -> Unit, ) { - val snackbarHostState = SnackbarHostState() + val navController = rememberNavController() + val recordState by viewModel.recordState.collectAsStateWithLifecycle() val selectedElementState by viewModel.selectedElementState.collectAsStateWithLifecycle() - InteractUiElementScreen( + val chooseAppState by viewModel.filteredAppListItems.collectAsStateWithLifecycle() + val appSearchQuery by viewModel.appSearchQuery.collectAsStateWithLifecycle() + + val onBackClick = { + if (!navController.navigateUp()) { + navigateBack() + } + } + + BackHandler(onBack = onBackClick) + + NavHost( modifier = modifier, - recordState = recordState, - selectedElementState = selectedElementState, - onBackClick = navigateBack, - onDoneClick = viewModel::onDoneClick, - onRecordClick = viewModel::onRecordClick, - snackbarHostState = snackbarHostState, - ) + navController = navController, + startDestination = DEST_LANDING, + enterTransition = { slideIntoContainer(towards = AnimatedContentTransitionScope.SlideDirection.Left) }, + exitTransition = { slideOutOfContainer(towards = AnimatedContentTransitionScope.SlideDirection.Right) }, + popEnterTransition = { slideIntoContainer(towards = AnimatedContentTransitionScope.SlideDirection.Right) }, + popExitTransition = { slideOutOfContainer(towards = AnimatedContentTransitionScope.SlideDirection.Right) }, + ) { + composable(DEST_LANDING) { + LandingScreen( + modifier = Modifier.fillMaxSize(), + recordState = recordState, + selectedElementState = selectedElementState, + onRecordClick = viewModel::onRecordClick, + onBackClick = onBackClick, + onDoneClick = viewModel::onDoneClick, + openSelectAppScreen = { + navController.navigate(DEST_SELECT_APP) + }, + ) + } + + composable(DEST_SELECT_APP) { + ChooseAppScreen( + modifier = Modifier.fillMaxSize(), + title = stringResource(R.string.action_interact_ui_element_choose_element_title), + state = chooseAppState, + query = appSearchQuery, + onQueryChange = { query -> viewModel.appSearchQuery.update { query } }, + onCloseSearch = { viewModel.appSearchQuery.update { null } }, + onNavigateBack = onBackClick, + ) + } + } } @Composable -private fun InteractUiElementScreen( +private fun LandingScreen( modifier: Modifier = Modifier, recordState: State, selectedElementState: SelectedUiElementState?, + onRecordClick: () -> Unit = {}, onBackClick: () -> Unit = {}, onDoneClick: () -> Unit = {}, - onRecordClick: () -> Unit = {}, - snackbarHostState: SnackbarHostState = SnackbarHostState(), + openSelectAppScreen: () -> Unit = {}, ) { - BackHandler(onBack = onBackClick) + val snackbarHostState = SnackbarHostState() Scaffold( modifier.displayCutoutPadding(), @@ -126,7 +174,6 @@ private fun InteractUiElementScreen( start = startPadding, end = endPadding, ), - ) { Column { Text( @@ -156,6 +203,7 @@ private fun InteractUiElementScreen( .padding(horizontal = 16.dp), state = recordState, onRecordClick = onRecordClick, + openSelectAppScreen = openSelectAppScreen, ) Spacer(modifier = Modifier.height(8.dp)) @@ -173,6 +221,7 @@ private fun RecordingSection( modifier: Modifier = Modifier, state: State, onRecordClick: () -> Unit = {}, + openSelectAppScreen: () -> Unit = {}, ) { Column(modifier = modifier) { when (state) { @@ -186,7 +235,7 @@ private fun RecordingSection( InteractionCountBox( modifier = Modifier.fillMaxWidth(), interactionCount = interactionCount, - onClick = {}, + onClick = openSelectAppScreen, ) Spacer(modifier = Modifier.height(8.dp)) @@ -319,7 +368,7 @@ private fun SelectedElementSection(modifier: Modifier = Modifier, state: Selecte @Composable private fun PreviewEmpty() { KeyMapperTheme { - InteractUiElementScreen( + LandingScreen( recordState = State.Data(RecordUiElementState.Empty), selectedElementState = null, ) @@ -332,7 +381,7 @@ private fun PreviewSelectedElement() { val appIcon = LocalContext.current.drawable(R.mipmap.ic_launcher_round) KeyMapperTheme { - InteractUiElementScreen( + LandingScreen( recordState = State.Data(RecordUiElementState.Recorded(3)), selectedElementState = SelectedUiElementState( description = "Test", @@ -353,7 +402,7 @@ private fun PreviewSelectedElement() { @Composable private fun PreviewLoading() { KeyMapperTheme { - InteractUiElementScreen( + LandingScreen( recordState = State.Loading, selectedElementState = null, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt index 014731b87b..7f8a6de072 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -1,6 +1,8 @@ package io.github.sds100.keymapper.actions.uielement import android.view.accessibility.AccessibilityNodeInfo +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.rounded.Android import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.viewModelScope @@ -9,15 +11,22 @@ import io.github.sds100.keymapper.actions.ActionData import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState import io.github.sds100.keymapper.util.Error import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.Success +import io.github.sds100.keymapper.util.containsQuery import io.github.sds100.keymapper.util.dataOrNull import io.github.sds100.keymapper.util.ifIsData +import io.github.sds100.keymapper.util.mapData import io.github.sds100.keymapper.util.onFailure +import io.github.sds100.keymapper.util.otherwise +import io.github.sds100.keymapper.util.then import io.github.sds100.keymapper.util.ui.PopupViewModel import io.github.sds100.keymapper.util.ui.PopupViewModelImpl import io.github.sds100.keymapper.util.ui.ResourceProvider import io.github.sds100.keymapper.util.ui.ViewModelHelper import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow @@ -25,6 +34,7 @@ import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch @@ -72,6 +82,22 @@ class InteractUiElementViewModel( val selectedElementState: StateFlow = _selectedElementState.asStateFlow() + val appSearchQuery = MutableStateFlow(null) + + private val appListItems: Flow>> = useCase.interactedPackages + .map { state -> state.mapData(::createInteractedPackageListItems) } + + val filteredAppListItems = combine( + appListItems, + appSearchQuery, + ) { state, query -> + state.mapData { listItems -> + listItems.filter { model -> + model.title.containsQuery(query) + } + } + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + fun loadAction(action: ActionData.InteractUiElement) { viewModelScope.launch { val appName = useCase.getAppName(action.packageName).valueOrNull() ?: action.packageName @@ -145,6 +171,23 @@ class InteractUiElementViewModel( } } + private fun createInteractedPackageListItems(packages: List): List { + return packages.map { packageName -> + val appName = useCase.getAppName(packageName).valueOrNull() ?: packageName + val appIcon = useCase + .getAppIcon(packageName) + .then { Success(ComposeIconInfo.Drawable(it)) } + .otherwise { Success(ComposeIconInfo.Vector(Icons.Rounded.Android)) } + .valueOrNull()!! + + SimpleListItemModel( + id = packageName, + title = appName, + icon = appIcon, + ) + } + } + private fun getNodeActionName(nodeAction: Int): String { return when (nodeAction) { AccessibilityNodeInfo.ACTION_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_click) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt b/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt new file mode 100644 index 0000000000..ca05260be4 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt @@ -0,0 +1,216 @@ +package io.github.sds100.keymapper.system.apps + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.calculateEndPadding +import androidx.compose.foundation.layout.calculateStartPadding +import androidx.compose.foundation.layout.displayCutoutPadding +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.drawable +import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions +import io.github.sds100.keymapper.util.ui.compose.SimpleListItem +import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel + +@Composable +fun ChooseAppScreen( + modifier: Modifier = Modifier, + title: String, + state: State>, + query: String? = null, + onQueryChange: (String) -> Unit = {}, + onCloseSearch: () -> Unit = {}, + onNavigateBack: () -> Unit = {}, + onClickApp: (String) -> Unit = {}, +) { + Scaffold( + modifier.displayCutoutPadding(), + bottomBar = { + BottomAppBar( + modifier = Modifier.imePadding(), + actions = { + SearchAppBarActions( + onCloseSearch = onCloseSearch, + onNavigateBack = onNavigateBack, + onQueryChange = onQueryChange, + enabled = state is State.Data, + query = query, + ) + }, + ) + }, + ) { innerPadding -> + val layoutDirection = LocalLayoutDirection.current + val startPadding = innerPadding.calculateStartPadding(layoutDirection) + val endPadding = innerPadding.calculateEndPadding(layoutDirection) + + Surface( + modifier = Modifier + .fillMaxSize() + .padding( + top = innerPadding.calculateTopPadding(), + bottom = innerPadding.calculateBottomPadding(), + start = startPadding, + end = endPadding, + ), + ) { + Column { + Text( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = 16.dp, + bottom = 8.dp, + ), + text = title, + style = MaterialTheme.typography.titleLarge, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + when (state) { + State.Loading -> LoadingScreen(modifier = Modifier.fillMaxSize()) + is State.Data -> { + val items = state.data + + if (items.isEmpty()) { + EmptyScreen(modifier = Modifier.fillMaxSize()) + } else { + ListScreen( + modifier = Modifier.fillMaxSize(), + listItems = items, + onClick = onClickApp, + ) + } + } + } + } + } + } +} + +@Composable +private fun LoadingScreen(modifier: Modifier = Modifier) { + Box(modifier) { + CircularProgressIndicator(modifier = Modifier.align(Alignment.Center)) + } +} + +@Composable +private fun EmptyScreen(modifier: Modifier = Modifier) { + Box(modifier) { + val shrug = stringResource(R.string.shrug) + val text = stringResource(R.string.app_list_empty) + Text( + modifier = Modifier.align(Alignment.Center), + text = buildAnnotatedString { + withStyle(MaterialTheme.typography.headlineLarge.toSpanStyle()) { + append(shrug) + } + appendLine() + appendLine() + withStyle(MaterialTheme.typography.bodyLarge.toSpanStyle()) { + append(text) + } + }, + textAlign = TextAlign.Center, + ) + } +} + +@Composable +private fun ListScreen( + modifier: Modifier = Modifier, + listItems: List, + onClick: (String) -> Unit, +) { + LazyColumn( + modifier = modifier, + contentPadding = PaddingValues(8.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + items(listItems) { model -> + SimpleListItem( + modifier = Modifier.fillMaxWidth(), + model = model, + onClick = { onClick(model.id) }, + ) + } + } +} + +@Preview +@Composable +private fun Empty() { + KeyMapperTheme { + ChooseAppScreen(title = "Choose app", state = State.Data(emptyList())) + } +} + +@Preview +@Composable +private fun Loading() { + KeyMapperTheme { + ChooseAppScreen(title = "Choose app", state = State.Loading) + } +} + +@Preview +@Composable +private fun Loaded() { + val icon = LocalContext.current.drawable(R.mipmap.ic_launcher_round) + + KeyMapperTheme { + ChooseAppScreen( + title = "Choose app", + state = State.Data( + listOf( + SimpleListItemModel( + id = "1", + title = "Key Mapper", + icon = ComposeIconInfo.Drawable(icon), + ), + SimpleListItemModel( + id = "2", + title = "Key Mapper", + icon = ComposeIconInfo.Drawable(icon), + ), + SimpleListItemModel( + id = "3", + title = "Key Mapper", + icon = ComposeIconInfo.Drawable(icon), + ), + ), + ), + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt new file mode 100644 index 0000000000..5ee078990d --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt @@ -0,0 +1,90 @@ +package io.github.sds100.keymapper.util.ui.compose + +import androidx.compose.foundation.layout.RowScope +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.rounded.ArrowBack +import androidx.compose.material.icons.rounded.Search +import androidx.compose.material3.DockedSearchBar +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.SearchBarDefaults +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import io.github.sds100.keymapper.R + +@Composable +@OptIn(ExperimentalMaterial3Api::class) +fun RowScope.SearchAppBarActions( + onCloseSearch: () -> Unit, + onNavigateBack: () -> Unit, + onQueryChange: (String) -> Unit, + enabled: Boolean, + query: String?, +) { + var isExpanded: Boolean by rememberSaveable { mutableStateOf(false) } + + IconButton(onClick = { + if (isExpanded) { + onCloseSearch() + isExpanded = false + } else { + onNavigateBack() + } + }) { + Icon( + Icons.AutoMirrored.Rounded.ArrowBack, + contentDescription = stringResource(R.string.bottom_app_bar_back_content_description), + ) + } + + DockedSearchBar( + modifier = Modifier.Companion.align(Alignment.Companion.CenterVertically), + inputField = { + SearchBarDefaults.InputField( + modifier = Modifier.Companion.align(Alignment.Companion.CenterVertically), + onSearch = { + onQueryChange(it) + isExpanded = false + }, + leadingIcon = { + Icon( + Icons.Rounded.Search, + contentDescription = null, + ) + }, + enabled = enabled, + placeholder = { Text(stringResource(R.string.search_placeholder)) }, + query = query ?: "", + onQueryChange = onQueryChange, + expanded = isExpanded, + onExpandedChange = { expanded -> + if (expanded) { + isExpanded = true + } else { + onCloseSearch() + isExpanded = false + } + }, + ) + }, + // This is false to prevent an empty "content" showing underneath. + expanded = isExpanded, + onExpandedChange = { expanded -> + if (expanded) { + isExpanded = true + } else { + onCloseSearch() + isExpanded = false + } + }, + content = {}, + ) +} diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt index d3a31360bc..0c758c99e7 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt @@ -1,11 +1,14 @@ package io.github.sds100.keymapper.util.ui.compose +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.IntrinsicSize import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width @@ -13,11 +16,13 @@ import androidx.compose.material.icons.Icons import androidx.compose.material.icons.rounded.Wifi import androidx.compose.material3.Icon import androidx.compose.material3.LocalContentColor +import androidx.compose.material3.LocalMinimumInteractiveComponentSize import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedCard import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color @@ -46,58 +51,86 @@ fun SimpleListItemHeader( } @Composable -fun SimpleListItem( +fun SimpleListItemFixedHeight( modifier: Modifier = Modifier, model: SimpleListItemModel, onClick: () -> Unit = {}, ) { - OutlinedCard(modifier = modifier.height(48.dp), onClick = onClick, enabled = model.isEnabled) { - Row(modifier = Modifier.fillMaxSize(), verticalAlignment = Alignment.CenterVertically) { - Spacer(modifier = Modifier.width(16.dp)) + SimpleListItem( + modifier = modifier.height(56.dp), + model = model, + onClick = onClick, + ) +} - when (model.icon) { - is ComposeIconInfo.Vector -> Icon( - modifier = Modifier.size(26.dp), - imageVector = model.icon.imageVector, - contentDescription = null, - tint = LocalContentColor.current, - ) +@Composable +fun SimpleListItem( + modifier: Modifier = Modifier, + model: SimpleListItemModel, + onClick: () -> Unit = {}, +) { + CompositionLocalProvider( + LocalMinimumInteractiveComponentSize provides 16.dp, + ) { + OutlinedCard( + modifier = modifier.height(IntrinsicSize.Min), + onClick = onClick, + enabled = model.isEnabled, + ) { + Row( + modifier = Modifier + .fillMaxSize(), + verticalAlignment = Alignment.CenterVertically, + ) { + Spacer(modifier = Modifier.width(16.dp)) - is ComposeIconInfo.Drawable -> { - val painter = rememberDrawablePainter(model.icon.drawable) - Icon( + when (model.icon) { + is ComposeIconInfo.Vector -> Icon( modifier = Modifier.size(26.dp), - painter = painter, + imageVector = model.icon.imageVector, contentDescription = null, - tint = Color.Unspecified, + tint = LocalContentColor.current, ) - } - } - Spacer(modifier = Modifier.width(16.dp)) + is ComposeIconInfo.Drawable -> { + val painter = rememberDrawablePainter(model.icon.drawable) + Icon( + modifier = Modifier.size(26.dp), + painter = painter, + contentDescription = null, + tint = Color.Unspecified, + ) + } + } - Column( - modifier = Modifier.padding(end = 16.dp), - ) { - Text( - text = model.title, - style = MaterialTheme.typography.bodyMedium, - maxLines = if (model.subtitle == null) { - 2 - } else { - 1 - }, - overflow = TextOverflow.Ellipsis, - ) + Spacer(modifier = Modifier.width(16.dp)) - if (model.subtitle != null) { + Column( + modifier = Modifier + .padding(end = 16.dp) + .heightIn(min = 36.dp), + verticalArrangement = Arrangement.Center, + ) { Text( - text = model.subtitle, - color = if (model.isSubtitleError) MaterialTheme.colorScheme.error else MaterialTheme.colorScheme.onSurface, - style = MaterialTheme.typography.bodySmall, - maxLines = 1, + text = model.title, + style = MaterialTheme.typography.bodyMedium, + maxLines = if (model.subtitle == null) { + 2 + } else { + 1 + }, overflow = TextOverflow.Ellipsis, ) + + if (model.subtitle != null) { + Text( + text = model.subtitle, + color = if (model.isSubtitleError) MaterialTheme.colorScheme.error else MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.bodySmall, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + ) + } } } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index beb6c3c235..915b5572fb 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -24,7 +24,7 @@ Enable accessibility service Restart accessibility service Share - + Nothing here! Stop repeating when… Trigger is released Trigger is pressed again From 410c1ae0f7c81a7735d9984c8f4aa3cc28f64ebb Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 13:33:46 +0200 Subject: [PATCH 10/21] #257 listing recorded UI elements works --- .../sds100/keymapper/actions/ActionData.kt | 5 +- .../actions/ActionDataEntityMapper.kt | 22 +- .../actions/PerformActionsUseCase.kt | 2 +- .../uielement/ChooseUiElementScreen.kt | 325 ++++++++++++++++++ .../uielement/InteractUiElementScreen.kt | 30 +- .../uielement/InteractUiElementUseCase.kt | 5 + .../uielement/InteractUiElementViewModel.kt | 151 ++++++-- .../actions/uielement/NodeInteractionType.kt | 14 + .../data/entities/AccessibilityNodeEntity.kt | 5 +- .../AccessibilityNodeRepository.kt | 24 +- .../AccessibilityNodeRecorder.kt | 14 +- app/src/main/res/values/strings.xml | 3 +- 12 files changed, 539 insertions(+), 61 deletions(-) create mode 100644 app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt index f2d4e8faa1..9b36222391 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt @@ -1,6 +1,7 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.display.Orientation import io.github.sds100.keymapper.system.intents.IntentExtraModel @@ -849,7 +850,7 @@ sealed class ActionData : Comparable { @Serializable data class InteractUiElement( val description: String, - val nodeAction: Int, + val nodeAction: NodeInteractionType, val packageName: String, val text: String?, val contentDescription: String?, @@ -859,7 +860,7 @@ sealed class ActionData : Comparable { /** * A list of the allowed accessibility node actions. */ - val nodeActions: List, + val nodeActions: List, ) : ActionData() { override val id: ActionId = ActionId.INTERACT_UI_ELEMENT } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index bcb0d4f8fc..77f6b92d87 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -1,6 +1,7 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.data.db.typeconverter.ConstantTypeConverters import io.github.sds100.keymapper.data.entities.ActionEntity import io.github.sds100.keymapper.data.entities.EntityExtra @@ -12,6 +13,8 @@ import io.github.sds100.keymapper.system.network.HttpMethod import io.github.sds100.keymapper.system.volume.DndMode import io.github.sds100.keymapper.system.volume.RingerMode import io.github.sds100.keymapper.system.volume.VolumeStream +import io.github.sds100.keymapper.util.Error +import io.github.sds100.keymapper.util.Result import io.github.sds100.keymapper.util.Success import io.github.sds100.keymapper.util.getKey import io.github.sds100.keymapper.util.success @@ -269,7 +272,7 @@ object ActionDataEntityMapper { ActionId.VOLUME_TOGGLE_MUTE, ActionId.VOLUME_UNMUTE, ActionId.VOLUME_MUTE, - -> { + -> { val showVolumeUi = entity.flags.hasFlag(ActionEntity.ACTION_FLAG_SHOW_VOLUME_UI) @@ -549,14 +552,17 @@ object ActionDataEntityMapper { entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID).valueOrNull() val actions = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS).then { - val intList = it.split(",").map { action -> action.toInt() } - Success(intList) + val typeList = it + .split(",") + .mapNotNull { action -> convertNodeInteractionType(it).valueOrNull() } + + Success(typeList) }.valueOrNull() ?: emptyList() val nodeAction = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_NODE_ACTION).then { - Success(it.toInt()) - }.valueOrNull()!! + convertNodeInteractionType(it) + }.valueOrNull() ?: return null ActionData.InteractUiElement( description = entity.data, @@ -573,6 +579,12 @@ object ActionDataEntityMapper { } } + private fun convertNodeInteractionType(string: String): Result = try { + Success(NodeInteractionType.valueOf(string)) + } catch (e: IllegalArgumentException) { + Error.Exception(e) + } + fun toEntity(data: ActionData): ActionEntity { val type = when (data) { is ActionData.Intent -> ActionEntity.Type.INTENT diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt index 3418b6edbc..0bb4dc9a59 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt @@ -811,7 +811,7 @@ class PerformActionsUseCaseImpl( // TODO compare other values node.uniqueId == action.uniqueId }, - performAction = { AccessibilityNodeAction(action = action.nodeAction) }, + performAction = { AccessibilityNodeAction(action = action.nodeAction.accessibilityActionId) }, ) } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt new file mode 100644 index 0000000000..d9a111ffc9 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt @@ -0,0 +1,325 @@ +package io.github.sds100.keymapper.actions.uielement + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.calculateEndPadding +import androidx.compose.foundation.layout.calculateStartPadding +import androidx.compose.foundation.layout.displayCutoutPadding +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.rounded.ErrorOutline +import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedCard +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions + +@Composable +fun ChooseElementScreen( + modifier: Modifier = Modifier, + state: State>, + query: String?, + onCloseSearch: () -> Unit = {}, + onNavigateBack: () -> Unit = {}, + onQueryChange: (String) -> Unit = {}, + onClickElement: (Long) -> Unit = {}, +) { + Scaffold( + modifier.displayCutoutPadding(), + bottomBar = { + BottomAppBar( + modifier = Modifier.imePadding(), + actions = { + SearchAppBarActions( + onCloseSearch = onCloseSearch, + onNavigateBack = onNavigateBack, + onQueryChange = onQueryChange, + enabled = state is State.Data, + query = query, + ) + }, + ) + }, + ) { innerPadding -> + val layoutDirection = LocalLayoutDirection.current + val startPadding = innerPadding.calculateStartPadding(layoutDirection) + val endPadding = innerPadding.calculateEndPadding(layoutDirection) + + Surface( + modifier = Modifier + .fillMaxSize() + .padding( + top = innerPadding.calculateTopPadding(), + bottom = innerPadding.calculateBottomPadding(), + start = startPadding, + end = endPadding, + ), + ) { + Column { + Text( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = 16.dp, + bottom = 8.dp, + ), + text = stringResource(R.string.action_interact_ui_element_choose_element_title), + style = MaterialTheme.typography.titleLarge, + ) + + Text( + modifier = Modifier.padding(horizontal = 16.dp), + text = stringResource(R.string.action_interact_ui_element_choose_element_text), + style = MaterialTheme.typography.bodyMedium, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Row( + modifier = Modifier.padding(horizontal = 16.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon( + imageVector = Icons.Rounded.ErrorOutline, + contentDescription = null, + tint = MaterialTheme.colorScheme.error, + ) + Spacer(modifier = Modifier.width(8.dp)) + Text( + text = stringResource(R.string.action_interact_ui_element_choose_element_not_found_subtitle), + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.titleSmall, + ) + } + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + modifier = Modifier.padding(horizontal = 16.dp), + text = stringResource(R.string.action_interact_ui_element_choose_element_not_found_text), + style = MaterialTheme.typography.bodyMedium, + ) + + // TODO key mapper dropdown menu + + when (state) { + State.Loading -> LoadingList(modifier = Modifier.fillMaxSize()) + is State.Data -> { + val items = state.data + + if (items.isEmpty()) { + EmptyList(modifier = Modifier.fillMaxSize()) + } else { + LoadedList( + modifier = Modifier.fillMaxSize(), + listItems = items, + onClick = onClickElement, + ) + } + } + } + } + } + } +} + +@Composable +private fun LoadingList(modifier: Modifier = Modifier) { + Box(modifier) { + CircularProgressIndicator(modifier = Modifier.align(Alignment.Center)) + } +} + +@Composable +private fun EmptyList(modifier: Modifier = Modifier) { + Box(modifier) { + val shrug = stringResource(R.string.shrug) + val text = stringResource(R.string.ui_element_list_empty) + Text( + modifier = Modifier.align(Alignment.Center), + text = buildAnnotatedString { + withStyle(MaterialTheme.typography.headlineLarge.toSpanStyle()) { + append(shrug) + } + appendLine() + appendLine() + withStyle(MaterialTheme.typography.bodyLarge.toSpanStyle()) { + append(text) + } + }, + textAlign = TextAlign.Center, + ) + } +} + +@Composable +private fun LoadedList( + modifier: Modifier = Modifier, + listItems: List, + onClick: (Long) -> Unit, +) { + LazyColumn( + modifier = modifier, + contentPadding = PaddingValues(8.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + items(listItems, key = { it.id }) { model -> + UiElementListItem( + modifier = Modifier.fillMaxWidth(), + model = model, + onClick = { onClick(model.id) }, + ) + } + } +} + +@Composable +private fun UiElementListItem( + modifier: Modifier = Modifier, + model: UiElementListItemModel, + onClick: () -> Unit, +) { + OutlinedCard(modifier = modifier, onClick = onClick) { + Column( + modifier = Modifier.padding(16.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + if (model.nodeViewResourceId != null) { + Text( + text = model.nodeViewResourceId, + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.primary, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + + if (model.nodeText != null) { + Text( + text = model.nodeText, + style = MaterialTheme.typography.titleMedium.copy(fontWeight = FontWeight.Bold), + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + + if (model.nodeClassName != null) { + TextWithLeadingLabel( + title = stringResource(R.string.action_interact_ui_element_class_name_label), + text = model.nodeClassName, + ) + } + + if (model.nodeUniqueId != null) { + TextWithLeadingLabel( + title = stringResource(R.string.action_interact_ui_element_unique_id_label), + text = model.nodeUniqueId, + ) + } + + TextWithLeadingLabel( + title = stringResource(R.string.action_interact_ui_element_interaction_types_label), + text = model.interactionTypes, + ) + } + } +} + +@Composable +private fun TextWithLeadingLabel( + modifier: Modifier = Modifier, + title: String, + text: String, +) { + val text = buildAnnotatedString { + pushStyle( + MaterialTheme.typography.labelLarge.copy(fontWeight = FontWeight.Bold).toSpanStyle(), + ) + append(title) + pop() + append(": ") + append(text) + } + + Text( + modifier = modifier, + text = text, + style = MaterialTheme.typography.bodySmall, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) +} + +@Preview +@Composable +private fun Empty() { + KeyMapperTheme { + ChooseElementScreen( + state = State.Data(emptyList()), + query = "Key Mapper", + ) + } +} + +@Preview +@Composable +private fun Loading() { + KeyMapperTheme { + ChooseElementScreen( + state = State.Loading, + query = null, + ) + } +} + +@Preview +@Composable +private fun Loaded() { + KeyMapperTheme { + ChooseElementScreen( + state = State.Data( + listOf( + UiElementListItemModel( + id = 1L, + nodeText = "Open Settings", + nodeClassName = "android.widget.ImageButton", + nodeViewResourceId = "menu_button", + nodeUniqueId = "123456789", + interactionTypes = "Tap, Tap and hold, Scroll forward", + ), + ), + ), + query = "Key Mapper", + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index b6c94c3943..cac5489f21 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -76,9 +76,12 @@ fun InteractUiElementScreen( val recordState by viewModel.recordState.collectAsStateWithLifecycle() val selectedElementState by viewModel.selectedElementState.collectAsStateWithLifecycle() - val chooseAppState by viewModel.filteredAppListItems.collectAsStateWithLifecycle() + val appListState by viewModel.filteredAppListItems.collectAsStateWithLifecycle() val appSearchQuery by viewModel.appSearchQuery.collectAsStateWithLifecycle() + val elementListState by viewModel.filteredElementListItems.collectAsStateWithLifecycle() + val elementSearchQuery by viewModel.elementSearchQuery.collectAsStateWithLifecycle() + val onBackClick = { if (!navController.navigateUp()) { navigateBack() @@ -114,11 +117,30 @@ fun InteractUiElementScreen( ChooseAppScreen( modifier = Modifier.fillMaxSize(), title = stringResource(R.string.action_interact_ui_element_choose_element_title), - state = chooseAppState, + state = appListState, query = appSearchQuery, onQueryChange = { query -> viewModel.appSearchQuery.update { query } }, onCloseSearch = { viewModel.appSearchQuery.update { null } }, onNavigateBack = onBackClick, + onClickApp = { + viewModel.onSelectApp(it) + navController.navigate(DEST_SELECT_ELEMENT) + }, + ) + } + + composable(DEST_SELECT_ELEMENT) { + ChooseElementScreen( + modifier = Modifier.fillMaxSize(), + state = elementListState, + query = elementSearchQuery, + onCloseSearch = { viewModel.elementSearchQuery.update { null } }, + onNavigateBack = onBackClick, + onQueryChange = { query -> viewModel.elementSearchQuery.update { query } }, + onClickElement = { + viewModel.onSelectElement(it) + navController.popBackStack(route = DEST_LANDING, inclusive = false) + }, ) } } @@ -388,11 +410,11 @@ private fun PreviewSelectedElement() { appName = "Test App", appIcon = ComposeIconInfo.Drawable(appIcon), nodeText = "Test Node", - nodeClassName = "Test Class", + nodeClassName = "android.widget.ImageButton", nodeViewResourceId = "io.github.sds100.keymapper:id/menu_button", nodeUniqueId = "123", interactionTypes = listOf(), - selectedInteraction = 0, + selectedInteraction = NodeInteractionType.LONG_CLICK, ), ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt index 79688b1629..c1553c6291 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt @@ -54,6 +54,10 @@ class InteractUiElementController( } } + override suspend fun getInteractionById(id: Long): AccessibilityNodeEntity? { + return nodeRepository.get(id) + } + override fun getAppName(packageName: String): Result = packageManagerAdapter.getAppName(packageName) override fun getAppIcon(packageName: String): Result = packageManagerAdapter.getAppIcon(packageName) @@ -79,6 +83,7 @@ interface InteractUiElementUseCase { val interactionCount: Flow> val interactedPackages: Flow>> fun getInteractionsByPackage(packageName: String): Flow>> + suspend fun getInteractionById(id: Long): AccessibilityNodeEntity? fun getAppName(packageName: String): Result fun getAppIcon(packageName: String): Result diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt index 7f8a6de072..d16db3acf6 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -1,6 +1,5 @@ package io.github.sds100.keymapper.actions.uielement -import android.view.accessibility.AccessibilityNodeInfo import androidx.compose.material.icons.Icons import androidx.compose.material.icons.rounded.Android import androidx.lifecycle.ViewModel @@ -8,6 +7,7 @@ import androidx.lifecycle.ViewModelProvider import androidx.lifecycle.viewModelScope import io.github.sds100.keymapper.R import io.github.sds100.keymapper.actions.ActionData +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState import io.github.sds100.keymapper.util.Error import io.github.sds100.keymapper.util.State @@ -26,6 +26,7 @@ import io.github.sds100.keymapper.util.ui.ViewModelHelper import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow @@ -34,6 +35,8 @@ import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.update @@ -85,7 +88,7 @@ class InteractUiElementViewModel( val appSearchQuery = MutableStateFlow(null) private val appListItems: Flow>> = useCase.interactedPackages - .map { state -> state.mapData(::createInteractedPackageListItems) } + .map { state -> state.mapData { list -> list.map(::buildInteractedPackageListItem) } } val filteredAppListItems = combine( appListItems, @@ -98,11 +101,39 @@ class InteractUiElementViewModel( } }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + private val selectedApp = MutableStateFlow(null) + + val elementSearchQuery = MutableStateFlow(null) + + @OptIn(ExperimentalCoroutinesApi::class) + private val elementListItems: StateFlow>> = selectedApp + .filterNotNull() + .flatMapLatest { packageName -> useCase.getInteractionsByPackage(packageName) } + .map { state -> state.mapData { list -> list.map(::buildUiElementListItem) } } + .stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + val filteredElementListItems = combine( + elementListItems, + elementSearchQuery, + ) { state, query -> + state.mapData { listItems -> + listItems.filter { model -> + val modelString = buildString { + append(model.nodeText) + append(" ") + append(model.nodeClassName) + append(" ") + append(model.nodeViewResourceId) + } + modelString.containsQuery(query) + } + } + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + fun loadAction(action: ActionData.InteractUiElement) { viewModelScope.launch { val appName = useCase.getAppName(action.packageName).valueOrNull() ?: action.packageName - val appIcon = useCase.getAppIcon(action.packageName).valueOrNull() - ?.let { ComposeIconInfo.Drawable(it) } + val appIcon = getAppIcon(action.packageName) val newState = SelectedUiElementState( description = action.description, @@ -153,6 +184,36 @@ class InteractUiElementViewModel( } } + fun onSelectApp(packageName: String) { + elementSearchQuery.update { null } + selectedApp.update { packageName } + } + + fun onSelectElement(id: Long) { + viewModelScope.launch { + val interaction = useCase.getInteractionById(id) ?: return@launch + + val appName = + useCase.getAppName(interaction.packageName).valueOrNull() ?: interaction.packageName + val appIcon = getAppIcon(interaction.packageName) + + val newState = SelectedUiElementState( + description = "", + appName = appName, + appIcon = appIcon, + nodeText = interaction.text ?: interaction.contentDescription, + nodeClassName = interaction.className, + nodeViewResourceId = interaction.viewResourceId, + nodeUniqueId = interaction.uniqueId, + interactionTypes = interaction.actions, + selectedInteraction = interaction.userInteractedActionId + ?: interaction.actions.first(), + ) + + _selectedElementState.update { newState } + } + } + private suspend fun startRecording() { useCase.startRecording().onFailure { error -> if (error == Error.AccessibilityServiceDisabled) { @@ -171,37 +232,46 @@ class InteractUiElementViewModel( } } - private fun createInteractedPackageListItems(packages: List): List { - return packages.map { packageName -> - val appName = useCase.getAppName(packageName).valueOrNull() ?: packageName - val appIcon = useCase - .getAppIcon(packageName) - .then { Success(ComposeIconInfo.Drawable(it)) } - .otherwise { Success(ComposeIconInfo.Vector(Icons.Rounded.Android)) } - .valueOrNull()!! - - SimpleListItemModel( - id = packageName, - title = appName, - icon = appIcon, - ) - } + private fun buildInteractedPackageListItem(packageName: String): SimpleListItemModel { + val appName = useCase.getAppName(packageName).valueOrNull() ?: packageName + val appIcon = getAppIcon(packageName) + + return SimpleListItemModel( + id = packageName, + title = appName, + icon = appIcon, + ) } - private fun getNodeActionName(nodeAction: Int): String { - return when (nodeAction) { - AccessibilityNodeInfo.ACTION_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_click) - AccessibilityNodeInfo.ACTION_LONG_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_long_click) - AccessibilityNodeInfo.ACTION_FOCUS -> getString(R.string.action_interact_ui_element_interaction_type_focus) - AccessibilityNodeInfo.ACTION_SELECT -> getString(R.string.action_interact_ui_element_interaction_type_select) - AccessibilityNodeInfo.ACTION_SCROLL_FORWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_forward) - AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_backward) - AccessibilityNodeInfo.ACTION_EXPAND -> getString(R.string.action_interact_ui_element_interaction_type_expand) - AccessibilityNodeInfo.ACTION_COLLAPSE -> getString(R.string.action_interact_ui_element_interaction_type_collapse) - else -> getString( - R.string.action_interact_ui_element_interaction_type_unknown, - nodeAction, - ) + private fun buildUiElementListItem(node: AccessibilityNodeEntity): UiElementListItemModel { + val resourceIdText = node.viewResourceId?.split("/")?.lastOrNull() + + return UiElementListItemModel( + id = node.id, + nodeViewResourceId = resourceIdText, + nodeText = node.text ?: node.contentDescription, + nodeClassName = node.className, + nodeUniqueId = node.uniqueId?.toString(), + interactionTypes = node.actions.joinToString { getInteractionTypeString(it) }, + ) + } + + private fun getAppIcon(packageName: String): ComposeIconInfo = useCase + .getAppIcon(packageName) + .then { Success(ComposeIconInfo.Drawable(it)) } + .otherwise { Success(ComposeIconInfo.Vector(Icons.Rounded.Android)) } + .valueOrNull()!! + + private fun getInteractionTypeString(interactionType: NodeInteractionType): String { + return when (interactionType) { + NodeInteractionType.CLICK -> getString(R.string.action_interact_ui_element_interaction_type_click) + NodeInteractionType.LONG_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_long_click) + NodeInteractionType.FOCUS -> getString(R.string.action_interact_ui_element_interaction_type_focus) + NodeInteractionType.SELECT -> getString(R.string.action_interact_ui_element_interaction_type_select) + NodeInteractionType.SCROLL_FORWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_forward) + NodeInteractionType.SCROLL_BACKWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_backward) + NodeInteractionType.EXPAND -> getString(R.string.action_interact_ui_element_interaction_type_expand) + NodeInteractionType.COLLAPSE -> getString(R.string.action_interact_ui_element_interaction_type_collapse) } } @@ -219,13 +289,13 @@ class InteractUiElementViewModel( data class SelectedUiElementState( val description: String, val appName: String, - val appIcon: ComposeIconInfo.Drawable?, + val appIcon: ComposeIconInfo, val nodeText: String?, val nodeClassName: String?, val nodeViewResourceId: String?, val nodeUniqueId: String?, - val interactionTypes: List, - val selectedInteraction: Int, + val interactionTypes: List, + val selectedInteraction: NodeInteractionType, ) sealed class RecordUiElementState { @@ -238,3 +308,12 @@ sealed class RecordUiElementState { data object Empty : RecordUiElementState() } + +data class UiElementListItemModel( + val id: Long, + val nodeViewResourceId: String?, + val nodeText: String?, + val nodeClassName: String?, + val nodeUniqueId: String?, + val interactionTypes: String, +) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt new file mode 100644 index 0000000000..f31a6520f5 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt @@ -0,0 +1,14 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.view.accessibility.AccessibilityNodeInfo + +enum class NodeInteractionType(val accessibilityActionId: Int) { + CLICK(AccessibilityNodeInfo.ACTION_CLICK), + LONG_CLICK(AccessibilityNodeInfo.ACTION_LONG_CLICK), + FOCUS(AccessibilityNodeInfo.ACTION_FOCUS), + SELECT(AccessibilityNodeInfo.ACTION_SELECT), + SCROLL_FORWARD(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD), + SCROLL_BACKWARD(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD), + EXPAND(AccessibilityNodeInfo.ACTION_EXPAND), + COLLAPSE(AccessibilityNodeInfo.ACTION_COLLAPSE), +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt index 8792942277..0913213f06 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -1,6 +1,7 @@ package io.github.sds100.keymapper.data.entities import android.os.Parcelable +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import kotlinx.parcelize.Parcelize import kotlinx.serialization.Serializable @@ -18,11 +19,11 @@ data class AccessibilityNodeEntity( /** * A list of the allowed accessibility node actions. */ - val actions: List, + val actions: List, /** * The accessibility action id of how the user interacted * with this node. This is null if the user didn't interact with * this node. */ - val userInteractedActionId: Int?, + val userInteractedActionId: NodeInteractionType?, ) : Parcelable diff --git a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt index be21fa91a6..ee0df25555 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt @@ -2,31 +2,41 @@ package io.github.sds100.keymapper.data.repositories import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.dataOrNull +import io.github.sds100.keymapper.util.mapData import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch interface AccessibilityNodeRepository { val nodes: Flow>> - fun insert(vararg node: AccessibilityNodeEntity) + fun get(id: Long): AccessibilityNodeEntity? + fun insert(node: AccessibilityNodeEntity) fun deleteAll() } class AccessibilityNodeRepositoryImpl(private val coroutineScope: CoroutineScope) : AccessibilityNodeRepository { - // TODO have a DAO to remember between app launches and so it isn't all cached in memory? + // TODO have a DAO to remember between app launches and so it isn't all cached in memory - also handles IDs automatically. + // TODO do not insert duplicates where all fields are the same override val nodes = MutableStateFlow>>(State.Data(ArrayList(128))) - override fun insert(vararg node: AccessibilityNodeEntity) { - coroutineScope.launch { - val currentState = nodes.value - if (currentState is State.Data) { - nodes.emit(State.Data(currentState.data.plus(node))) + override fun insert(node: AccessibilityNodeEntity) { + nodes.update { currentState -> + currentState.mapData { list -> + val nodeWithId = node.copy(id = list.size + 1L) + list.plus(nodeWithId) } } } + override fun get(id: Long): AccessibilityNodeEntity? { + val nodes = nodes.value.dataOrNull() ?: return null + return nodes.find { it.id == id } + } + override fun deleteAll() { coroutineScope.launch { nodes.emit(State.Data(ArrayList(128))) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt index d031696e04..678f3ec536 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.system.accessibility import android.os.Build import android.os.CountDownTimer import android.view.accessibility.AccessibilityEvent +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository import kotlinx.coroutines.flow.MutableStateFlow @@ -59,10 +60,17 @@ class AccessibilityNodeRecorder( val source = event.source ?: return - if (source.actionList.isNullOrEmpty()) { + val interactionTypes = source.actionList.mapNotNull { action -> + NodeInteractionType.entries.find { it.accessibilityActionId == action.id } + }.distinct() + + if (interactionTypes.isEmpty()) { return } + val userInteractedActionId = + NodeInteractionType.entries.find { it.accessibilityActionId == event.action } + val entity = AccessibilityNodeEntity( packageName = event.packageName.toString(), @@ -75,8 +83,8 @@ class AccessibilityNodeRecorder( } else { null }, - actions = source.actionList?.map { it.id } ?: emptyList(), - userInteractedActionId = event.action, + actions = interactionTypes, + userInteractedActionId = userInteractedActionId, ) nodeRepository.insert(entity) diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 915b5572fb..7fc86c0576 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -25,6 +25,7 @@ Restart accessibility service Share Nothing here! + Nothing here! Stop repeating when… Trigger is released Trigger is pressed again @@ -1131,9 +1132,9 @@ Text - View ID Class name Unique ID + Interaction types From 6824ed356bdf1f35669aa782e28a292ab8074f54 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 15:40:21 +0200 Subject: [PATCH 11/21] #257 filter interaction types --- .../actions/HttpRequestBottomSheet.kt | 10 +-- .../uielement/ChooseUiElementScreen.kt | 86 ++++++++++++++----- .../uielement/InteractUiElementScreen.kt | 3 +- .../uielement/InteractUiElementViewModel.kt | 70 +++++++++++++-- .../data/entities/AccessibilityNodeEntity.kt | 2 +- .../AccessibilityNodeRecorder.kt | 2 +- .../util/ui/compose/KeyMapperDropdownMenu.kt | 26 +++--- app/src/main/res/values/strings.xml | 1 + 8 files changed, 150 insertions(+), 50 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt b/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt index 25725b2a41..ee00632d66 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt @@ -48,7 +48,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull @OptIn(ExperimentalMaterial3Api::class) @Composable fun HttpRequestBottomSheet(delegate: CreateActionDelegate) { - val scope = rememberCoroutineScope() + rememberCoroutineScope() val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) if (delegate.httpRequestBottomSheetState != null) { @@ -139,10 +139,10 @@ private fun HttpRequestBottomSheet( .padding(horizontal = 16.dp), expanded = methodExpanded, onExpandedChange = { methodExpanded = it }, - value = state.method.toString(), - onValueChanged = { - onSelectMethod(HttpMethod.valueOf(it)) - }, + label = stringResource(R.string.action_http_request_method_label), + selectedValue = state.method, + values = HttpMethod.entries.map { it to it.toString() }, + onValueChanged = onSelectMethod, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt index d9a111ffc9..3d3355d28b 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt @@ -28,6 +28,10 @@ import androidx.compose.material3.Scaffold import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalLayoutDirection @@ -42,18 +46,22 @@ import androidx.compose.ui.unit.dp import io.github.sds100.keymapper.R import io.github.sds100.keymapper.compose.KeyMapperTheme import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.ui.compose.KeyMapperDropdownMenu import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions @Composable fun ChooseElementScreen( modifier: Modifier = Modifier, - state: State>, + state: State, query: String?, onCloseSearch: () -> Unit = {}, onNavigateBack: () -> Unit = {}, onQueryChange: (String) -> Unit = {}, onClickElement: (Long) -> Unit = {}, + onSelectInteractionType: (NodeInteractionType?) -> Unit = {}, ) { + var interactionTypeExpanded by rememberSaveable { mutableStateOf(false) } + Scaffold( modifier.displayCutoutPadding(), bottomBar = { @@ -129,20 +137,31 @@ fun ChooseElementScreen( text = stringResource(R.string.action_interact_ui_element_choose_element_not_found_text), style = MaterialTheme.typography.bodyMedium, ) - - // TODO key mapper dropdown menu + Spacer(modifier = Modifier.height(8.dp)) when (state) { State.Loading -> LoadingList(modifier = Modifier.fillMaxSize()) is State.Data -> { - val items = state.data + val listItems = state.data.listItems - if (items.isEmpty()) { + if (listItems.isEmpty()) { EmptyList(modifier = Modifier.fillMaxSize()) } else { + KeyMapperDropdownMenu( + modifier = Modifier.padding(horizontal = 16.dp), + expanded = interactionTypeExpanded, + onExpandedChange = { interactionTypeExpanded = it }, + label = stringResource(R.string.action_interact_ui_element_filter_interaction_type_dropdown), + values = state.data.interactionTypes, + selectedValue = state.data.selectedInteractionType, + onValueChanged = onSelectInteractionType, + ) + + Spacer(modifier = Modifier.height(8.dp)) + LoadedList( modifier = Modifier.fillMaxSize(), - listItems = items, + listItems = listItems, onClick = onClickElement, ) } @@ -216,7 +235,7 @@ private fun UiElementListItem( ) { if (model.nodeViewResourceId != null) { Text( - text = model.nodeViewResourceId, + text = "View ID: ${model.nodeViewResourceId}", style = MaterialTheme.typography.labelMedium, color = MaterialTheme.colorScheme.primary, maxLines = 2, @@ -226,7 +245,7 @@ private fun UiElementListItem( if (model.nodeText != null) { Text( - text = model.nodeText, + text = "\"${model.nodeText}\"", style = MaterialTheme.typography.titleMedium.copy(fontWeight = FontWeight.Bold), maxLines = 2, overflow = TextOverflow.Ellipsis, @@ -249,7 +268,7 @@ private fun UiElementListItem( TextWithLeadingLabel( title = stringResource(R.string.action_interact_ui_element_interaction_types_label), - text = model.interactionTypes, + text = model.interactionTypesText, ) } } @@ -263,7 +282,7 @@ private fun TextWithLeadingLabel( ) { val text = buildAnnotatedString { pushStyle( - MaterialTheme.typography.labelLarge.copy(fontWeight = FontWeight.Bold).toSpanStyle(), + MaterialTheme.typography.bodySmall.copy(fontWeight = FontWeight.Bold).toSpanStyle(), ) append(title) pop() @@ -285,7 +304,13 @@ private fun TextWithLeadingLabel( private fun Empty() { KeyMapperTheme { ChooseElementScreen( - state = State.Data(emptyList()), + state = State.Data( + SelectUiElementState( + listItems = emptyList(), + interactionTypes = emptyList(), + selectedInteractionType = null, + ), + ), query = "Key Mapper", ) } @@ -305,20 +330,35 @@ private fun Loading() { @Preview @Composable private fun Loaded() { + val listItems = listOf( + UiElementListItemModel( + id = 1L, + nodeText = "Open Settings", + nodeClassName = "android.widget.ImageButton", + nodeViewResourceId = "menu_button", + nodeUniqueId = "123456789", + interactionTypesText = "Tap, Tap and hold, Scroll forward", + interactionTypes = setOf( + NodeInteractionType.CLICK, + NodeInteractionType.LONG_CLICK, + NodeInteractionType.SCROLL_FORWARD, + ), + ), + ) + + val state = SelectUiElementState( + listItems = listItems, + interactionTypes = listOf( + null to "Any", + NodeInteractionType.CLICK to "Tap", + NodeInteractionType.LONG_CLICK to "Tap and hold", + ), + selectedInteractionType = null, + ) + KeyMapperTheme { ChooseElementScreen( - state = State.Data( - listOf( - UiElementListItemModel( - id = 1L, - nodeText = "Open Settings", - nodeClassName = "android.widget.ImageButton", - nodeViewResourceId = "menu_button", - nodeUniqueId = "123456789", - interactionTypes = "Tap, Tap and hold, Scroll forward", - ), - ), - ), + state = State.Data(state), query = "Key Mapper", ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index cac5489f21..6d9b6164af 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -79,7 +79,7 @@ fun InteractUiElementScreen( val appListState by viewModel.filteredAppListItems.collectAsStateWithLifecycle() val appSearchQuery by viewModel.appSearchQuery.collectAsStateWithLifecycle() - val elementListState by viewModel.filteredElementListItems.collectAsStateWithLifecycle() + val elementListState by viewModel.selectUiElementState.collectAsStateWithLifecycle() val elementSearchQuery by viewModel.elementSearchQuery.collectAsStateWithLifecycle() val onBackClick = { @@ -141,6 +141,7 @@ fun InteractUiElementScreen( viewModel.onSelectElement(it) navController.popBackStack(route = DEST_LANDING, inclusive = false) }, + onSelectInteractionType = viewModel::onSelectInteractionTypeFilter, ) } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt index d16db3acf6..23c995622a 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -106,18 +106,31 @@ class InteractUiElementViewModel( val elementSearchQuery = MutableStateFlow(null) @OptIn(ExperimentalCoroutinesApi::class) - private val elementListItems: StateFlow>> = selectedApp + private val interactionsByPackage: StateFlow>> = selectedApp .filterNotNull() .flatMapLatest { packageName -> useCase.getInteractionsByPackage(packageName) } - .map { state -> state.mapData { list -> list.map(::buildUiElementListItem) } } .stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) - val filteredElementListItems = combine( + private val elementListItems: Flow>> = interactionsByPackage + .map { state -> state.mapData { list -> list.map(::buildUiElementListItem) } } + + private val interactionTypesFilterItems: Flow>>> = + interactionsByPackage + .map { state -> state.mapData(::buildInteractionTypeFilterItems) } + + private val selectedInteractionTypeFilter = MutableStateFlow(null) + + private val filteredElementListItems = combine( elementListItems, elementSearchQuery, - ) { state, query -> + selectedInteractionTypeFilter, + ) { state, query, interactionType -> state.mapData { listItems -> listItems.filter { model -> + if (interactionType != null && !model.interactionTypes.contains(interactionType)) { + return@filter false + } + val modelString = buildString { append(model.nodeText) append(" ") @@ -130,6 +143,22 @@ class InteractUiElementViewModel( } }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + val selectUiElementState: StateFlow> = combine( + filteredElementListItems, + interactionTypesFilterItems, + selectedInteractionTypeFilter, + ) { listItemsState, interactionTypesState, selectedInteractionType -> + val listItems = listItemsState.dataOrNull() ?: return@combine State.Loading + val interactionTypes = interactionTypesState.dataOrNull() ?: return@combine State.Loading + + val newState = SelectUiElementState( + listItems = listItems, + interactionTypes = interactionTypes, + selectedInteractionType = selectedInteractionType, + ) + State.Data(newState) + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + fun loadAction(action: ActionData.InteractUiElement) { viewModelScope.launch { val appName = useCase.getAppName(action.packageName).valueOrNull() ?: action.packageName @@ -205,7 +234,7 @@ class InteractUiElementViewModel( nodeClassName = interaction.className, nodeViewResourceId = interaction.viewResourceId, nodeUniqueId = interaction.uniqueId, - interactionTypes = interaction.actions, + interactionTypes = interaction.actions.toList(), selectedInteraction = interaction.userInteractedActionId ?: interaction.actions.first(), ) @@ -214,6 +243,10 @@ class InteractUiElementViewModel( } } + fun onSelectInteractionTypeFilter(interactionType: NodeInteractionType?) { + selectedInteractionTypeFilter.update { interactionType } + } + private suspend fun startRecording() { useCase.startRecording().onFailure { error -> if (error == Error.AccessibilityServiceDisabled) { @@ -252,10 +285,26 @@ class InteractUiElementViewModel( nodeText = node.text ?: node.contentDescription, nodeClassName = node.className, nodeUniqueId = node.uniqueId?.toString(), - interactionTypes = node.actions.joinToString { getInteractionTypeString(it) }, + interactionTypesText = node.actions.joinToString { getInteractionTypeString(it) }, + interactionTypes = node.actions, ) } + private fun buildInteractionTypeFilterItems(nodes: List): List> { + val interactionTypes = nodes.flatMap { it.actions }.toSet() + + return buildList { + add(null to getString(R.string.action_interact_ui_element_interaction_type_any)) + + // They should always be in the same order so iterate over the Enum entries. + for (type in NodeInteractionType.entries) { + if (interactionTypes.contains(type)) { + add(type to getInteractionTypeString(type)) + } + } + } + } + private fun getAppIcon(packageName: String): ComposeIconInfo = useCase .getAppIcon(packageName) .then { Success(ComposeIconInfo.Drawable(it)) } @@ -309,11 +358,18 @@ sealed class RecordUiElementState { data object Empty : RecordUiElementState() } +data class SelectUiElementState( + val listItems: List, + val interactionTypes: List>, + val selectedInteractionType: NodeInteractionType?, +) + data class UiElementListItemModel( val id: Long, val nodeViewResourceId: String?, val nodeText: String?, val nodeClassName: String?, val nodeUniqueId: String?, - val interactionTypes: String, + val interactionTypesText: String, + val interactionTypes: Set, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt index 0913213f06..5a99a511ca 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -19,7 +19,7 @@ data class AccessibilityNodeEntity( /** * A list of the allowed accessibility node actions. */ - val actions: List, + val actions: Set, /** * The accessibility action id of how the user interacted * with this node. This is null if the user didn't interact with diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt index 678f3ec536..d8663c7153 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -83,7 +83,7 @@ class AccessibilityNodeRecorder( } else { null }, - actions = interactionTypes, + actions = interactionTypes.toSet(), userInteractedActionId = userInteractedActionId, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt index f293b15968..cddcfdcd78 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt @@ -10,18 +10,17 @@ import androidx.compose.material3.Text import androidx.compose.material3.TextField import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier -import androidx.compose.ui.res.stringResource -import io.github.sds100.keymapper.R -import io.github.sds100.keymapper.system.network.HttpMethod @Composable @OptIn(ExperimentalMaterial3Api::class) -fun KeyMapperDropdownMenu( +fun KeyMapperDropdownMenu( modifier: Modifier = Modifier, expanded: Boolean, onExpandedChange: (Boolean) -> Unit = {}, - value: String, - onValueChanged: (String) -> Unit = {}, + label: String, + selectedValue: T, + values: List>, + onValueChanged: (T) -> Unit = {}, ) { ExposedDropdownMenuBox( modifier = modifier, @@ -30,28 +29,31 @@ fun KeyMapperDropdownMenu( ) { TextField( modifier = Modifier.menuAnchor(MenuAnchorType.PrimaryNotEditable), - value = value, - onValueChange = onValueChanged, + value = values.single { it.first == selectedValue }.second, + onValueChange = { newValue -> + onValueChanged(values.single { it.second == newValue }.first) + }, readOnly = true, - label = { Text(stringResource(R.string.action_http_request_method_label)) }, + label = { Text(text = label) }, trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded) }, colors = ExposedDropdownMenuDefaults.textFieldColors(), ) + ExposedDropdownMenu( matchTextFieldWidth = true, expanded = expanded, onDismissRequest = { onExpandedChange(false) }, ) { - for (method in HttpMethod.entries) { + for ((value, valueText) in values) { DropdownMenuItem( text = { Text( - method.toString(), + valueText, style = MaterialTheme.typography.bodyLarge, ) }, onClick = { - onValueChanged(method.toString()) + onValueChanged(value) onExpandedChange(false) }, contentPadding = ExposedDropdownMenuDefaults.ItemContentPadding, diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7fc86c0576..fcdf7c1fe1 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1120,6 +1120,7 @@ Select how you want to interact with the UI element. Filter interaction type + Any Tap Tap and hold Focus From 3e1aba7bcef043d6084eff438b0e361f5ce17f9f Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 17:05:45 +0200 Subject: [PATCH 12/21] #257 complete creating interact UI element actions --- .../sds100/keymapper/actions/ActionData.kt | 2 +- .../actions/ActionDataEntityMapper.kt | 41 ++-- .../keymapper/actions/ActionUiHelper.kt | 2 +- .../sds100/keymapper/actions/ActionUtils.kt | 1 + .../keymapper/actions/CreateActionDelegate.kt | 6 +- .../actions/HttpRequestBottomSheet.kt | 2 +- .../uielement/ChooseUiElementScreen.kt | 2 +- .../uielement/InteractUiElementScreen.kt | 209 +++++++++++++++++- .../uielement/InteractUiElementViewModel.kt | 76 +++++-- .../keymapper/util/ui/NavigationViewModel.kt | 7 + .../util/ui/compose/KeyMapperDropdownMenu.kt | 6 +- app/src/main/res/values/strings.xml | 7 +- 12 files changed, 304 insertions(+), 57 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt index 9b36222391..3823a2e5f7 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt @@ -860,7 +860,7 @@ sealed class ActionData : Comparable { /** * A list of the allowed accessibility node actions. */ - val nodeActions: List, + val nodeActions: Set, ) : ActionData() { override val id: ActionId = ActionId.INTERACT_UI_ELEMENT } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index 77f6b92d87..3ae0464723 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -293,7 +293,7 @@ object ActionDataEntityMapper { ActionId.TOGGLE_FLASHLIGHT, ActionId.ENABLE_FLASHLIGHT, ActionId.CHANGE_FLASHLIGHT_STRENGTH, - -> { + -> { val lens = entity.extras.getData(ActionEntity.EXTRA_LENS).then { LENS_MAP.getKey(it)!!.success() }.valueOrNull() ?: return null @@ -552,12 +552,17 @@ object ActionDataEntityMapper { entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID).valueOrNull() val actions = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS).then { - val typeList = it - .split(",") - .mapNotNull { action -> convertNodeInteractionType(it).valueOrNull() } - - Success(typeList) - }.valueOrNull() ?: emptyList() + val nodeActionMask = it.toInt() + val interactionTypeSet = mutableSetOf() + + for (type in NodeInteractionType.entries) { + if (nodeActionMask and type.accessibilityActionId == type.accessibilityActionId) { + interactionTypeSet.add(type) + } + } + + Success(interactionTypeSet) + }.valueOrNull() ?: emptySet() val nodeAction = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_NODE_ACTION).then { @@ -820,14 +825,12 @@ object ActionDataEntityMapper { ), ) - data.packageName.let { - add( - EntityExtra( - ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME, - it, - ), - ) - } + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME, + data.packageName, + ), + ) data.contentDescription?.let { add( @@ -868,10 +871,16 @@ object ActionDataEntityMapper { } if (data.nodeActions.isNotEmpty()) { + var nodeActionMask = 0 + + for (nodeAction in data.nodeActions) { + nodeActionMask = nodeActionMask or nodeAction.accessibilityActionId + } + add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS, - data.nodeActions.joinToString(","), + nodeActionMask.toString(), ), ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt index e782916e4b..ccd1fac039 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt @@ -530,7 +530,7 @@ class ActionUiHelper( ActionData.DeviceControls -> getString(R.string.action_device_controls) is ActionData.HttpRequest -> action.description - is ActionData.InteractUiElement -> getString(R.string.action_interact_ui_element_title) + is ActionData.InteractUiElement -> action.description } fun getIcon(action: ActionData): ComposeIconInfo = when (action) { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt index 12b23dd67f..a2a0d82c73 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt @@ -827,6 +827,7 @@ fun ActionData.isEditable(): Boolean = when (this) { is ActionData.Url, is ActionData.PhoneCall, is ActionData.HttpRequest, + is ActionData.InteractUiElement, -> true else -> false diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt index ac94158e21..649ef42bd7 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt @@ -788,11 +788,7 @@ class CreateActionDelegate( } ActionId.INTERACT_UI_ELEMENT -> { - val oldAction = if (oldData is ActionData.InteractUiElement) { - oldData - } else { - null - } + val oldAction = oldData as? ActionData.InteractUiElement return navigate( "config_interact_ui_element_action", diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt b/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt index ee00632d66..15d0063245 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt @@ -139,7 +139,7 @@ private fun HttpRequestBottomSheet( .padding(horizontal = 16.dp), expanded = methodExpanded, onExpandedChange = { methodExpanded = it }, - label = stringResource(R.string.action_http_request_method_label), + label = { Text(stringResource(R.string.action_http_request_method_label)) }, selectedValue = state.method, values = HttpMethod.entries.map { it to it.toString() }, onValueChanged = onSelectMethod, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt index 3d3355d28b..612cb913af 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt @@ -151,7 +151,7 @@ fun ChooseElementScreen( modifier = Modifier.padding(horizontal = 16.dp), expanded = interactionTypeExpanded, onExpandedChange = { interactionTypeExpanded = it }, - label = stringResource(R.string.action_interact_ui_element_filter_interaction_type_dropdown), + label = { Text(stringResource(R.string.action_interact_ui_element_filter_interaction_type_dropdown)) }, values = state.data.interactionTypes, selectedValue = state.data.selectedInteractionType, onValueChanged = onSelectInteractionType, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index 6d9b6164af..946655a742 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.actions.uielement import androidx.activity.compose.BackHandler import androidx.compose.animation.AnimatedContentTransitionScope import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer @@ -13,9 +14,14 @@ import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.sizeIn import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.rounded.ArrowBack +import androidx.compose.material.icons.outlined.Info import androidx.compose.material.icons.rounded.Check import androidx.compose.material.icons.rounded.ChevronRight import androidx.compose.material3.BottomAppBar @@ -23,22 +29,29 @@ import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.ExtendedFloatingActionButton import androidx.compose.material3.FilledTonalButton import androidx.compose.material3.FloatingActionButtonDefaults +import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.LocalContentColor import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.Scaffold import androidx.compose.material3.SnackbarHost import androidx.compose.material3.SnackbarHostState import androidx.compose.material3.Surface import androidx.compose.material3.Text +import androidx.compose.material3.contentColorFor import androidx.compose.runtime.Composable import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalLayoutDirection import androidx.compose.ui.res.pluralStringResource @@ -50,6 +63,7 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.rememberNavController +import com.google.accompanist.drawablepainter.rememberDrawablePainter import io.github.sds100.keymapper.R import io.github.sds100.keymapper.compose.KeyMapperTheme import io.github.sds100.keymapper.compose.LocalCustomColorsPalette @@ -57,7 +71,10 @@ import io.github.sds100.keymapper.system.apps.ChooseAppScreen import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.drawable import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.KeyMapperDropdownMenu +import io.github.sds100.keymapper.util.ui.compose.OptionsHeaderRow import io.github.sds100.keymapper.util.ui.compose.icons.AdGroup +import io.github.sds100.keymapper.util.ui.compose.icons.JumpToElement import io.github.sds100.keymapper.util.ui.compose.icons.KeyMapperIcons import kotlinx.coroutines.flow.update @@ -110,6 +127,8 @@ fun InteractUiElementScreen( openSelectAppScreen = { navController.navigate(DEST_SELECT_APP) }, + onSelectInteractionType = viewModel::onSelectElementInteractionType, + onDescriptionChanged = viewModel::onDescriptionChanged, ) } @@ -156,6 +175,8 @@ private fun LandingScreen( onBackClick: () -> Unit = {}, onDoneClick: () -> Unit = {}, openSelectAppScreen: () -> Unit = {}, + onSelectInteractionType: (NodeInteractionType) -> Unit = {}, + onDescriptionChanged: (String) -> Unit = {}, ) { val snackbarHostState = SnackbarHostState() @@ -171,7 +192,12 @@ private fun LandingScreen( ) } }, floatingActionButton = { - if (selectedElementState != null) { + if (selectedElementState == null || selectedElementState.description.isBlank()) { + DisabledExtendedFloatingActionButton( + icon = { Icon(Icons.Rounded.Check, stringResource(R.string.button_done)) }, + text = stringResource(R.string.button_done), + ) + } else { ExtendedFloatingActionButton( onClick = onDoneClick, text = { Text(stringResource(R.string.button_done)) }, @@ -198,7 +224,7 @@ private fun LandingScreen( end = endPadding, ), ) { - Column { + Column(modifier = Modifier.verticalScroll(rememberScrollState())) { Text( modifier = Modifier.padding( start = 16.dp, @@ -232,13 +258,63 @@ private fun LandingScreen( Spacer(modifier = Modifier.height(8.dp)) if (selectedElementState != null) { - SelectedElementSection(modifier = Modifier.fillMaxWidth(), selectedElementState) + HorizontalDivider( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + ) + + Spacer(modifier = Modifier.height(8.dp)) + + SelectedElementSection( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + state = selectedElementState, + onSelectInteractionType = onSelectInteractionType, + onDescriptionChanged = onDescriptionChanged, + ) } } } } } +@Composable +private fun DisabledExtendedFloatingActionButton( + modifier: Modifier = Modifier, + icon: @Composable () -> Unit, + text: String, +) { + Surface( + modifier = modifier, + shape = FloatingActionButtonDefaults.extendedFabShape, + color = FloatingActionButtonDefaults.containerColor.copy(alpha = 0.5f), + ) { + Row( + modifier = + Modifier + .sizeIn(minWidth = 80.dp, minHeight = 56.dp) + .padding(start = 16.dp, end = 20.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.Start, + ) { + val contentColor = + MaterialTheme.colorScheme.contentColorFor(FloatingActionButtonDefaults.containerColor) + .copy(alpha = 0.5f) + + CompositionLocalProvider(LocalContentColor provides contentColor) { + icon() + Spacer(Modifier.width(12.dp)) + Text( + text, + style = MaterialTheme.typography.labelLarge, + ) + } + } + } +} + @Composable private fun RecordingSection( modifier: Modifier = Modifier, @@ -382,8 +458,129 @@ private fun RecordButton( } @Composable -private fun SelectedElementSection(modifier: Modifier = Modifier, state: SelectedUiElementState) { +private fun SelectedElementSection( + modifier: Modifier = Modifier, + state: SelectedUiElementState, + onDescriptionChanged: (String) -> Unit = {}, + onSelectInteractionType: (NodeInteractionType) -> Unit = {}, +) { + var interactionTypeExpanded by rememberSaveable { mutableStateOf(false) } + Column(modifier = modifier) { + val isError = state.description.isBlank() + + OutlinedTextField( + modifier = Modifier.fillMaxWidth(), + value = state.description, + onValueChange = onDescriptionChanged, + isError = isError, + supportingText = if (isError) { + { Text(stringResource(R.string.error_cant_be_empty)) } + } else { + null + }, + label = { + Text(stringResource(R.string.action_interact_ui_element_description_label)) + }, + ) + + Spacer(modifier = Modifier.height(16.dp)) + + OptionsHeaderRow( + icon = Icons.Outlined.Info, + text = stringResource(R.string.action_interact_ui_element_interaction_details_title), + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = stringResource(R.string.action_interact_ui_element_app_label), + style = MaterialTheme.typography.titleSmall, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) { + if (state.appIcon != null) { + val painter = rememberDrawablePainter(state.appIcon.drawable) + Icon( + modifier = Modifier.size(24.dp), + painter = painter, + contentDescription = null, + tint = Color.Unspecified, + ) + Spacer(modifier = Modifier.width(8.dp)) + Text(text = state.appName, style = MaterialTheme.typography.bodyMedium) + } + } + + Spacer(modifier = Modifier.height(8.dp)) + + if (state.nodeText != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_text_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeText, style = MaterialTheme.typography.bodyMedium) + } + + Spacer(modifier = Modifier.height(8.dp)) + + if (state.nodeClassName != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_class_name_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeClassName, style = MaterialTheme.typography.bodyMedium) + Spacer(modifier = Modifier.height(8.dp)) + } + + if (state.nodeViewResourceId != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_view_id_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeViewResourceId, style = MaterialTheme.typography.bodyMedium) + + Spacer(modifier = Modifier.height(8.dp)) + } + + if (state.nodeUniqueId != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_unique_id_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeUniqueId, style = MaterialTheme.typography.bodyMedium) + Spacer(modifier = Modifier.height(8.dp)) + } + + OptionsHeaderRow( + icon = KeyMapperIcons.JumpToElement, + text = stringResource(R.string.action_interact_ui_element_interaction_type_dropdown), + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = stringResource(R.string.action_interact_ui_element_interaction_type_dropdown_caption), + style = MaterialTheme.typography.bodyMedium, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + KeyMapperDropdownMenu( + expanded = interactionTypeExpanded, + onExpandedChange = { interactionTypeExpanded = it }, + values = state.interactionTypes, + selectedValue = state.selectedInteraction, + onValueChanged = onSelectInteractionType, + ) + + Spacer(modifier = Modifier.height(8.dp)) } } @@ -407,14 +604,14 @@ private fun PreviewSelectedElement() { LandingScreen( recordState = State.Data(RecordUiElementState.Recorded(3)), selectedElementState = SelectedUiElementState( - description = "Test", + description = "Tap test node", appName = "Test App", appIcon = ComposeIconInfo.Drawable(appIcon), nodeText = "Test Node", nodeClassName = "android.widget.ImageButton", nodeViewResourceId = "io.github.sds100.keymapper:id/menu_button", nodeUniqueId = "123", - interactionTypes = listOf(), + interactionTypes = listOf(NodeInteractionType.LONG_CLICK to "Tap and hold"), selectedInteraction = NodeInteractionType.LONG_CLICK, ), ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt index 23c995622a..db28e58093 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -17,7 +17,6 @@ import io.github.sds100.keymapper.util.dataOrNull import io.github.sds100.keymapper.util.ifIsData import io.github.sds100.keymapper.util.mapData import io.github.sds100.keymapper.util.onFailure -import io.github.sds100.keymapper.util.otherwise import io.github.sds100.keymapper.util.then import io.github.sds100.keymapper.util.ui.PopupViewModel import io.github.sds100.keymapper.util.ui.PopupViewModelImpl @@ -33,6 +32,7 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.filterNotNull @@ -41,6 +41,7 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch +import java.util.Locale class InteractUiElementViewModel( private val useCase: InteractUiElementUseCase, @@ -50,7 +51,7 @@ class InteractUiElementViewModel( PopupViewModel by PopupViewModelImpl() { private val _returnAction: MutableSharedFlow = MutableSharedFlow() - val returnAction: SharedFlow = _returnAction + val returnAction: SharedFlow = _returnAction.asSharedFlow() val recordState: StateFlow> = combine( useCase.recordState, @@ -63,9 +64,16 @@ class InteractUiElementViewModel( val mins = recordState.timeLeft / 60 val secs = recordState.timeLeft % 60 + val timeRemainingText = String.format( + Locale.getDefault(), + "%02d:%02d", + mins, + secs, + ) + State.Data( RecordUiElementState.CountingDown( - timeRemaining = "$mins:$secs", + timeRemaining = timeRemainingText, interactionCount = interactionCount, ), ) @@ -116,7 +124,18 @@ class InteractUiElementViewModel( private val interactionTypesFilterItems: Flow>>> = interactionsByPackage - .map { state -> state.mapData(::buildInteractionTypeFilterItems) } + .map { state -> + state.mapData { list -> + val any = Pair( + null, + getString(R.string.action_interact_ui_element_interaction_type_any), + ) + + val interactionTypes = list.flatMap { it.actions }.toSet() + + listOf(any).plus(buildInteractionTypeFilterItems(interactionTypes)) + } + } private val selectedInteractionTypeFilter = MutableStateFlow(null) @@ -172,7 +191,7 @@ class InteractUiElementViewModel( nodeClassName = action.className, nodeViewResourceId = action.viewResourceId, nodeUniqueId = action.uniqueId, - interactionTypes = action.nodeActions, + interactionTypes = buildInteractionTypeFilterItems(action.nodeActions), selectedInteraction = action.nodeAction, ) @@ -186,6 +205,10 @@ class InteractUiElementViewModel( return } + if (selectedElementState.description.isBlank()) { + return + } + val action = ActionData.InteractUiElement( description = selectedElementState.description, nodeAction = selectedElementState.selectedInteraction, @@ -195,10 +218,12 @@ class InteractUiElementViewModel( className = selectedElementState.nodeClassName, viewResourceId = selectedElementState.nodeViewResourceId, uniqueId = selectedElementState.nodeUniqueId, - nodeActions = selectedElementState.interactionTypes, + nodeActions = selectedElementState.interactionTypes.map { it.first }.toSet(), ) - _returnAction.tryEmit(action) + viewModelScope.launch { + _returnAction.emit(action) + } } fun onRecordClick() { @@ -226,6 +251,9 @@ class InteractUiElementViewModel( useCase.getAppName(interaction.packageName).valueOrNull() ?: interaction.packageName val appIcon = getAppIcon(interaction.packageName) + val selectedInteraction = + NodeInteractionType.entries.first { interaction.actions.contains(it) } + val newState = SelectedUiElementState( description = "", appName = appName, @@ -234,19 +262,30 @@ class InteractUiElementViewModel( nodeClassName = interaction.className, nodeViewResourceId = interaction.viewResourceId, nodeUniqueId = interaction.uniqueId, - interactionTypes = interaction.actions.toList(), - selectedInteraction = interaction.userInteractedActionId - ?: interaction.actions.first(), + interactionTypes = buildInteractionTypeFilterItems(interaction.actions), + selectedInteraction = selectedInteraction, ) _selectedElementState.update { newState } } } + fun onSelectElementInteractionType(interactionType: NodeInteractionType) { + _selectedElementState.update { state -> + state?.copy(selectedInteraction = interactionType) + } + } + fun onSelectInteractionTypeFilter(interactionType: NodeInteractionType?) { selectedInteractionTypeFilter.update { interactionType } } + fun onDescriptionChanged(description: String) { + _selectedElementState.update { state -> + state?.copy(description = description) + } + } + private suspend fun startRecording() { useCase.startRecording().onFailure { error -> if (error == Error.AccessibilityServiceDisabled) { @@ -267,7 +306,7 @@ class InteractUiElementViewModel( private fun buildInteractedPackageListItem(packageName: String): SimpleListItemModel { val appName = useCase.getAppName(packageName).valueOrNull() ?: packageName - val appIcon = getAppIcon(packageName) + val appIcon = getAppIcon(packageName) ?: ComposeIconInfo.Vector(Icons.Rounded.Android) return SimpleListItemModel( id = packageName, @@ -290,12 +329,8 @@ class InteractUiElementViewModel( ) } - private fun buildInteractionTypeFilterItems(nodes: List): List> { - val interactionTypes = nodes.flatMap { it.actions }.toSet() - + private fun buildInteractionTypeFilterItems(interactionTypes: Set): List> { return buildList { - add(null to getString(R.string.action_interact_ui_element_interaction_type_any)) - // They should always be in the same order so iterate over the Enum entries. for (type in NodeInteractionType.entries) { if (interactionTypes.contains(type)) { @@ -305,11 +340,10 @@ class InteractUiElementViewModel( } } - private fun getAppIcon(packageName: String): ComposeIconInfo = useCase + private fun getAppIcon(packageName: String): ComposeIconInfo.Drawable? = useCase .getAppIcon(packageName) .then { Success(ComposeIconInfo.Drawable(it)) } - .otherwise { Success(ComposeIconInfo.Vector(Icons.Rounded.Android)) } - .valueOrNull()!! + .valueOrNull() private fun getInteractionTypeString(interactionType: NodeInteractionType): String { return when (interactionType) { @@ -338,12 +372,12 @@ class InteractUiElementViewModel( data class SelectedUiElementState( val description: String, val appName: String, - val appIcon: ComposeIconInfo, + val appIcon: ComposeIconInfo.Drawable?, val nodeText: String?, val nodeClassName: String?, val nodeViewResourceId: String?, val nodeUniqueId: String?, - val interactionTypes: List, + val interactionTypes: List>, val selectedInteraction: NodeInteractionType, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt index 67d57171b5..fc9c239439 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt @@ -20,6 +20,7 @@ import io.github.sds100.keymapper.actions.swipescreen.SwipePickCoordinateResult import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateFragment import io.github.sds100.keymapper.actions.tapscreen.PickCoordinateResult import io.github.sds100.keymapper.actions.tapscreen.PickDisplayCoordinateFragment +import io.github.sds100.keymapper.actions.uielement.InteractUiElementFragment import io.github.sds100.keymapper.constraints.ChooseConstraintFragment import io.github.sds100.keymapper.constraints.Constraint import io.github.sds100.keymapper.system.apps.ActivityInfo @@ -330,5 +331,11 @@ fun NavigationViewModel.sendNavResultFromBundle( onNavResult(NavResult(requestKey, BluetoothDeviceInfo(address, name))) } + + NavDestination.ID_INTERACT_UI_ELEMENT_ACTION -> { + val json = bundle.getString(InteractUiElementFragment.EXTRA_ACTION)!! + val result = Json.decodeFromString(json) + onNavResult(NavResult(requestKey, result)) + } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt index cddcfdcd78..eafba75c5d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt @@ -17,7 +17,7 @@ fun KeyMapperDropdownMenu( modifier: Modifier = Modifier, expanded: Boolean, onExpandedChange: (Boolean) -> Unit = {}, - label: String, + label: (@Composable () -> Unit)? = null, selectedValue: T, values: List>, onValueChanged: (T) -> Unit = {}, @@ -29,12 +29,12 @@ fun KeyMapperDropdownMenu( ) { TextField( modifier = Modifier.menuAnchor(MenuAnchorType.PrimaryNotEditable), - value = values.single { it.first == selectedValue }.second, + value = values.find { it.first == selectedValue }?.second ?: values.first().second, onValueChange = { newValue -> onValueChanged(values.single { it.second == newValue }.first) }, readOnly = true, - label = { Text(text = label) }, + label = label, trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded) }, colors = ExposedDropdownMenuDefaults.textFieldColors(), ) diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index fcdf7c1fe1..587ccb6ecd 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1131,9 +1131,12 @@ Collapse Unknown: %d - Text - + Interaction details + Description + App + Text / content description Class name + View resource ID Unique ID Interaction types From 670631e594a706e4da000520548f8e7604cc2bb2 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 17:24:21 +0200 Subject: [PATCH 13/21] #257 performing interact with UI element actions works --- .../actions/ConfigActionsViewModel.kt | 1 + .../actions/PerformActionsUseCase.kt | 57 ++++++++++++++++--- .../uielement/InteractUiElementScreen.kt | 1 + .../uielement/InteractUiElementViewModel.kt | 5 +- .../sds100/keymapper/util/ErrorUtils.kt | 1 + .../io/github/sds100/keymapper/util/Result.kt | 2 + app/src/main/res/values/strings.xml | 2 + 7 files changed, 59 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt index f501900c70..96d6da8bb0 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt @@ -177,6 +177,7 @@ class ConfigActionsViewModel( override fun onEditClick() { val actionUid = actionOptionsUid.value ?: return coroutineScope.launch { + actionOptionsUid.update { null } val keyMap = config.keyMap.first().dataOrNull() ?: return@launch val oldAction = keyMap.actionList.find { it.uid == actionUid } ?: return@launch diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt index 0bb4dc9a59..80c6ea900d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt @@ -10,6 +10,7 @@ import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.PreferenceDefaults import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeAction +import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeModel import io.github.sds100.keymapper.system.accessibility.IAccessibilityService import io.github.sds100.keymapper.system.accessibility.ServiceAdapter import io.github.sds100.keymapper.system.airplanemode.AirplaneModeAdapter @@ -803,16 +804,15 @@ class PerformActionsUseCaseImpl( is ActionData.InteractUiElement -> { if (accessibilityService.activeWindowPackage.first() != action.packageName) { - // TODO + result = Error.UiElementNotFound + } else { + result = accessibilityService.performActionOnNode( + findNode = { node -> + matchAccessibilityNode(node, action) + }, + performAction = { AccessibilityNodeAction(action = action.nodeAction.accessibilityActionId) }, + ) } - - result = accessibilityService.performActionOnNode( - findNode = { node -> - // TODO compare other values - node.uniqueId == action.uniqueId - }, - performAction = { AccessibilityNodeAction(action = action.nodeAction.accessibilityActionId) }, - ) } } @@ -906,6 +906,45 @@ class PerformActionsUseCaseImpl( popupMessageAdapter.showPopupMessage(it.getFullMessage(resourceProvider)) } } + + private fun matchAccessibilityNode( + node: AccessibilityNodeModel, + action: ActionData.InteractUiElement, + ): Boolean { + if (compareIfNonNull(node.uniqueId, action.uniqueId)) { + return true + } + + val viewResourceIdMatches = node.viewResourceId == action.viewResourceId + val classNameMatches = node.className == action.className + + if (compareIfNonNull( + node.contentDescription, + action.contentDescription, + ) && + viewResourceIdMatches && + classNameMatches + ) { + return true + } + + if (compareIfNonNull(node.text, action.text) && + viewResourceIdMatches && + classNameMatches + ) { + return true + } + + if (viewResourceIdMatches) { + return true + } + + return false + } + + private fun compareIfNonNull(a: T?, b: T?): Boolean { + return a != null && b != null && a == b + } } interface PerformActionsUseCase { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt index 946655a742..c4fd683a5d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -605,6 +605,7 @@ private fun PreviewSelectedElement() { recordState = State.Data(RecordUiElementState.Recorded(3)), selectedElementState = SelectedUiElementState( description = "Tap test node", + packageName = "com.example.test", appName = "Test App", appIcon = ComposeIconInfo.Drawable(appIcon), nodeText = "Test Node", diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt index db28e58093..80d255273b 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -185,6 +185,7 @@ class InteractUiElementViewModel( val newState = SelectedUiElementState( description = action.description, + packageName = action.packageName, appName = appName, appIcon = appIcon, nodeText = action.text ?: action.contentDescription, @@ -212,7 +213,7 @@ class InteractUiElementViewModel( val action = ActionData.InteractUiElement( description = selectedElementState.description, nodeAction = selectedElementState.selectedInteraction, - packageName = selectedElementState.appName, + packageName = selectedElementState.packageName, text = selectedElementState.nodeText, contentDescription = selectedElementState.nodeText, className = selectedElementState.nodeClassName, @@ -256,6 +257,7 @@ class InteractUiElementViewModel( val newState = SelectedUiElementState( description = "", + packageName = interaction.packageName, appName = appName, appIcon = appIcon, nodeText = interaction.text ?: interaction.contentDescription, @@ -371,6 +373,7 @@ class InteractUiElementViewModel( data class SelectedUiElementState( val description: String, + val packageName: String, val appName: String, val appIcon: ComposeIconInfo.Drawable?, val nodeText: String?, diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt b/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt index 4aa7e57171..8a047e25b0 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt @@ -162,6 +162,7 @@ fun Error.getFullMessage(resourceProvider: ResourceProvider): String = when (thi Error.DpadTriggerImeNotSelected -> resourceProvider.getString(R.string.trigger_error_dpad_ime_not_selected) Error.InvalidBackup -> resourceProvider.getString(R.string.error_invalid_backup) Error.MalformedUrl -> resourceProvider.getString(R.string.error_malformed_url) + Error.UiElementNotFound -> resourceProvider.getString(R.string.error_ui_element_not_found) } val Error.isFixable: Boolean diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Result.kt b/app/src/main/java/io/github/sds100/keymapper/util/Result.kt index 0d904a6685..8fe8251769 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Result.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Result.kt @@ -145,6 +145,8 @@ sealed class Error : Result() { */ data object DpadTriggerImeNotSelected : Error() data object MalformedUrl : Error() + + data object UiElementNotFound : Error() } inline fun Result.onSuccess(f: (T) -> Unit): Result { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 587ccb6ecd..933bc823e1 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -909,6 +909,8 @@ Must be greater than 0! Must be greater than 0! Must be %d or less! + + UI element not found! From 829634054ce35d0d4753e9a786e95ff4258bee03 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 20:28:19 +0200 Subject: [PATCH 14/21] #257 save nodes to the database --- .../19.json | 441 ++++++++++++++++++ .../github/sds100/keymapper/ServiceLocator.kt | 1 + .../actions/ActionDataEntityMapper.kt | 22 +- .../uielement/InteractUiElementUseCase.kt | 1 + .../sds100/keymapper/data/db/AppDatabase.kt | 12 +- .../data/db/dao/AccessibilityNodeDao.kt | 35 ++ .../NodeInteractionTypeSetTypeConverter.kt | 34 ++ .../data/entities/AccessibilityNodeEntity.kt | 39 +- .../data/migration/AutoMigration18To19.kt | 5 + .../AccessibilityNodeRepository.kt | 61 ++- .../AccessibilityNodeRecorder.kt | 52 ++- 11 files changed, 629 insertions(+), 74 deletions(-) create mode 100644 app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json create mode 100644 app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt create mode 100644 app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt diff --git a/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json b/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json new file mode 100644 index 0000000000..ed9aef420f --- /dev/null +++ b/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json @@ -0,0 +1,441 @@ +{ + "formatVersion": 1, + "database": { + "version": 19, + "identityHash": "300aa53acf7905efdf0c5b0ff7516ec9", + "entities": [ + { + "tableName": "keymaps", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `trigger` TEXT NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `flags` INTEGER NOT NULL, `is_enabled` INTEGER NOT NULL, `uid` TEXT NOT NULL, `group_uid` TEXT, FOREIGN KEY(`group_uid`) REFERENCES `groups`(`uid`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "trigger", + "columnName": "trigger", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "actionList", + "columnName": "action_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraint_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isEnabled", + "columnName": "is_enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "groupUid", + "columnName": "group_uid", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_keymaps_uid", + "unique": true, + "columnNames": [ + "uid" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_keymaps_uid` ON `${TABLE_NAME}` (`uid`)" + } + ], + "foreignKeys": [ + { + "table": "groups", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "group_uid" + ], + "referencedColumns": [ + "uid" + ] + } + ] + }, + { + "tableName": "fingerprintmaps", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `extras` TEXT NOT NULL, `flags` INTEGER NOT NULL, `is_enabled` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionList", + "columnName": "action_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraint_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "extras", + "columnName": "extras", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isEnabled", + "columnName": "is_enabled", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `time` INTEGER NOT NULL, `severity` INTEGER NOT NULL, `message` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "time", + "columnName": "time", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "severity", + "columnName": "severity", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "message", + "columnName": "message", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "floating_layouts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` TEXT NOT NULL, `name` TEXT NOT NULL, PRIMARY KEY(`uid`))", + "fields": [ + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uid" + ] + }, + "indices": [ + { + "name": "index_floating_layouts_name", + "unique": true, + "columnNames": [ + "name" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_floating_layouts_name` ON `${TABLE_NAME}` (`name`)" + } + ] + }, + { + "tableName": "floating_buttons", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` TEXT NOT NULL, `layout_uid` TEXT NOT NULL, `text` TEXT NOT NULL, `button_size` INTEGER NOT NULL, `x` INTEGER NOT NULL, `y` INTEGER NOT NULL, `orientation` TEXT NOT NULL, `display_width` INTEGER NOT NULL, `display_height` INTEGER NOT NULL, `border_opacity` REAL, `background_opacity` REAL, PRIMARY KEY(`uid`), FOREIGN KEY(`layout_uid`) REFERENCES `floating_layouts`(`uid`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "layoutUid", + "columnName": "layout_uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "buttonSize", + "columnName": "button_size", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "x", + "columnName": "x", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "y", + "columnName": "y", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "orientation", + "columnName": "orientation", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayWidth", + "columnName": "display_width", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "displayHeight", + "columnName": "display_height", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "borderOpacity", + "columnName": "border_opacity", + "affinity": "REAL" + }, + { + "fieldPath": "backgroundOpacity", + "columnName": "background_opacity", + "affinity": "REAL" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uid" + ] + }, + "indices": [ + { + "name": "index_floating_buttons_layout_uid", + "unique": false, + "columnNames": [ + "layout_uid" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_floating_buttons_layout_uid` ON `${TABLE_NAME}` (`layout_uid`)" + } + ], + "foreignKeys": [ + { + "table": "floating_layouts", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "layout_uid" + ], + "referencedColumns": [ + "uid" + ] + } + ] + }, + { + "tableName": "groups", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` TEXT NOT NULL, `name` TEXT NOT NULL, `constraints` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `parent_uid` TEXT, `last_opened_date` INTEGER, PRIMARY KEY(`uid`), FOREIGN KEY(`parent_uid`) REFERENCES `groups`(`uid`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraints", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "parentUid", + "columnName": "parent_uid", + "affinity": "TEXT" + }, + { + "fieldPath": "lastOpenedDate", + "columnName": "last_opened_date", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uid" + ] + }, + "foreignKeys": [ + { + "table": "groups", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "parent_uid" + ], + "referencedColumns": [ + "uid" + ] + } + ] + }, + { + "tableName": "accessibility_nodes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `package_name` TEXT NOT NULL, `text` TEXT, `content_description` TEXT, `class_name` TEXT, `view_resource_id` TEXT, `unique_id` TEXT, `actions` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "package_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT" + }, + { + "fieldPath": "contentDescription", + "columnName": "content_description", + "affinity": "TEXT" + }, + { + "fieldPath": "className", + "columnName": "class_name", + "affinity": "TEXT" + }, + { + "fieldPath": "viewResourceId", + "columnName": "view_resource_id", + "affinity": "TEXT" + }, + { + "fieldPath": "uniqueId", + "columnName": "unique_id", + "affinity": "TEXT" + }, + { + "fieldPath": "actions", + "columnName": "actions", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '300aa53acf7905efdf0c5b0ff7516ec9')" + ] + } +} \ No newline at end of file diff --git a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt index 342bd175ce..c0dd401af4 100755 --- a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt +++ b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt @@ -223,6 +223,7 @@ object ServiceLocator { synchronized(this) { return accessibilityNodeRepository ?: AccessibilityNodeRepositoryImpl( (context.applicationContext as KeyMapperApp).appCoroutineScope, + database(context).accessibilityNodeDao(), ).also { this.accessibilityNodeRepository = it } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index 3ae0464723..b6b99e1726 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.data.db.typeconverter.ConstantTypeConverters +import io.github.sds100.keymapper.data.db.typeconverter.NodeInteractionTypeSetTypeConverter import io.github.sds100.keymapper.data.entities.ActionEntity import io.github.sds100.keymapper.data.entities.EntityExtra import io.github.sds100.keymapper.data.entities.getData @@ -319,7 +320,7 @@ object ActionDataEntityMapper { } ActionId.DISABLE_FLASHLIGHT, - -> { + -> { val lens = entity.extras.getData(ActionEntity.EXTRA_LENS).then { LENS_MAP.getKey(it)!!.success() }.valueOrNull() ?: return null @@ -552,16 +553,7 @@ object ActionDataEntityMapper { entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID).valueOrNull() val actions = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS).then { - val nodeActionMask = it.toInt() - val interactionTypeSet = mutableSetOf() - - for (type in NodeInteractionType.entries) { - if (nodeActionMask and type.accessibilityActionId == type.accessibilityActionId) { - interactionTypeSet.add(type) - } - } - - Success(interactionTypeSet) + Success(NodeInteractionTypeSetTypeConverter().toSet(it.toInt())) }.valueOrNull() ?: emptySet() val nodeAction = @@ -871,16 +863,10 @@ object ActionDataEntityMapper { } if (data.nodeActions.isNotEmpty()) { - var nodeActionMask = 0 - - for (nodeAction in data.nodeActions) { - nodeActionMask = nodeActionMask or nodeAction.accessibilityActionId - } - add( EntityExtra( ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS, - nodeActionMask.toString(), + NodeInteractionTypeSetTypeConverter().toMask(data.nodeActions).toString(), ), ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt index c1553c6291..9a325f22f3 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt @@ -63,6 +63,7 @@ class InteractUiElementController( override fun getAppIcon(packageName: String): Result = packageManagerAdapter.getAppIcon(packageName) override suspend fun startRecording(): Result<*> { + nodeRepository.deleteAll() return serviceAdapter.send(ServiceEvent.StartRecordingNodes) } diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt index e3a933e747..e4644b1055 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt @@ -9,6 +9,7 @@ import androidx.room.TypeConverters import androidx.room.migration.Migration import androidx.sqlite.db.SupportSQLiteDatabase import io.github.sds100.keymapper.data.db.AppDatabase.Companion.DATABASE_VERSION +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao import io.github.sds100.keymapper.data.db.dao.FingerprintMapDao import io.github.sds100.keymapper.data.db.dao.FloatingButtonDao import io.github.sds100.keymapper.data.db.dao.FloatingLayoutDao @@ -18,7 +19,9 @@ import io.github.sds100.keymapper.data.db.dao.LogEntryDao import io.github.sds100.keymapper.data.db.typeconverter.ActionListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.ConstraintListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.ExtraListTypeConverter +import io.github.sds100.keymapper.data.db.typeconverter.NodeInteractionTypeSetTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.TriggerTypeConverter +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.data.entities.FingerprintMapEntity import io.github.sds100.keymapper.data.entities.FloatingButtonEntity import io.github.sds100.keymapper.data.entities.FloatingLayoutEntity @@ -28,6 +31,7 @@ import io.github.sds100.keymapper.data.entities.LogEntryEntity import io.github.sds100.keymapper.data.migration.AutoMigration14To15 import io.github.sds100.keymapper.data.migration.AutoMigration15To16 import io.github.sds100.keymapper.data.migration.AutoMigration16To17 +import io.github.sds100.keymapper.data.migration.AutoMigration18To19 import io.github.sds100.keymapper.data.migration.Migration10To11 import io.github.sds100.keymapper.data.migration.Migration11To12 import io.github.sds100.keymapper.data.migration.Migration13To14 @@ -44,7 +48,7 @@ import io.github.sds100.keymapper.data.migration.Migration9To10 * Created by sds100 on 24/01/2020. */ @Database( - entities = [KeyMapEntity::class, FingerprintMapEntity::class, LogEntryEntity::class, FloatingLayoutEntity::class, FloatingButtonEntity::class, GroupEntity::class], + entities = [KeyMapEntity::class, FingerprintMapEntity::class, LogEntryEntity::class, FloatingLayoutEntity::class, FloatingButtonEntity::class, GroupEntity::class, AccessibilityNodeEntity::class], version = DATABASE_VERSION, exportSchema = true, autoMigrations = [ @@ -54,6 +58,8 @@ import io.github.sds100.keymapper.data.migration.Migration9To10 AutoMigration(from = 15, to = 16, spec = AutoMigration15To16::class), // This adds last opened timestamp to groups AutoMigration(from = 16, to = 17, spec = AutoMigration16To17::class), + // Adds accessibility node table + AutoMigration(from = 18, to = 19, spec = AutoMigration18To19::class), ], ) @TypeConverters( @@ -61,11 +67,12 @@ import io.github.sds100.keymapper.data.migration.Migration9To10 ExtraListTypeConverter::class, TriggerTypeConverter::class, ConstraintListTypeConverter::class, + NodeInteractionTypeSetTypeConverter::class, ) abstract class AppDatabase : RoomDatabase() { companion object { const val DATABASE_NAME = "key_map_database" - const val DATABASE_VERSION = 18 + const val DATABASE_VERSION = 19 val MIGRATION_1_2 = object : Migration(1, 2) { @@ -162,4 +169,5 @@ abstract class AppDatabase : RoomDatabase() { abstract fun floatingLayoutDao(): FloatingLayoutDao abstract fun floatingButtonDao(): FloatingButtonDao abstract fun groupDao(): GroupDao + abstract fun accessibilityNodeDao(): AccessibilityNodeDao } diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt new file mode 100644 index 0000000000..b86cf032bd --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt @@ -0,0 +1,35 @@ +package io.github.sds100.keymapper.data.db.dao + +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import kotlinx.coroutines.flow.Flow + +@Dao +interface AccessibilityNodeDao { + companion object { + const val TABLE_NAME = "accessibility_nodes" + const val KEY_ID = "id" + const val KEY_PACKAGE_NAME = "package_name" + const val KEY_TEXT = "text" + const val KEY_CONTENT_DESCRIPTION = "content_description" + const val KEY_CLASS_NAME = "class_name" + const val KEY_VIEW_RESOURCE_ID = "view_resource_id" + const val KEY_UNIQUE_ID = "unique_id" + const val KEY_ACTIONS = "actions" + } + + @Query("SELECT * FROM $TABLE_NAME WHERE $KEY_ID = (:id)") + suspend fun getById(id: Long): AccessibilityNodeEntity? + + @Query("SELECT * FROM $TABLE_NAME") + fun getAll(): Flow> + + @Insert(onConflict = OnConflictStrategy.ABORT) + suspend fun insert(vararg node: AccessibilityNodeEntity) + + @Query("DELETE FROM $TABLE_NAME") + suspend fun deleteAll() +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt new file mode 100644 index 0000000000..d4093d82d8 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt @@ -0,0 +1,34 @@ +package io.github.sds100.keymapper.data.db.typeconverter + +import androidx.room.TypeConverter +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType + +/** + * Created by sds100 on 05/09/2018. + */ + +class NodeInteractionTypeSetTypeConverter { + @TypeConverter + fun toSet(mask: Int): Set { + val interactionTypeSet = mutableSetOf() + + for (type in NodeInteractionType.entries) { + if (mask and type.accessibilityActionId == type.accessibilityActionId) { + interactionTypeSet.add(type) + } + } + + return interactionTypeSet + } + + @TypeConverter + fun toMask(set: Set): Int { + var nodeActionMask = 0 + + for (nodeAction in set) { + nodeActionMask = nodeActionMask or nodeAction.accessibilityActionId + } + + return nodeActionMask + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt index 5a99a511ca..896a12e8a5 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -1,29 +1,48 @@ package io.github.sds100.keymapper.data.entities import android.os.Parcelable +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.PrimaryKey import io.github.sds100.keymapper.actions.uielement.NodeInteractionType +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_ACTIONS +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_CLASS_NAME +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_CONTENT_DESCRIPTION +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_ID +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_PACKAGE_NAME +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_TEXT +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_UNIQUE_ID +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_VIEW_RESOURCE_ID +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.TABLE_NAME import kotlinx.parcelize.Parcelize import kotlinx.serialization.Serializable @Serializable @Parcelize +@Entity(tableName = TABLE_NAME,) data class AccessibilityNodeEntity( + @PrimaryKey(autoGenerate = true) + @ColumnInfo(name = KEY_ID) val id: Long = 0L, - val parentId: Long? = null, + + @ColumnInfo(name = KEY_PACKAGE_NAME) val packageName: String, + + @ColumnInfo(name = KEY_TEXT) val text: String?, + + @ColumnInfo(name = KEY_CONTENT_DESCRIPTION) val contentDescription: String?, + + @ColumnInfo(name = KEY_CLASS_NAME) val className: String?, + + @ColumnInfo(name = KEY_VIEW_RESOURCE_ID) val viewResourceId: String?, + + @ColumnInfo(name = KEY_UNIQUE_ID) val uniqueId: String?, - /** - * A list of the allowed accessibility node actions. - */ + + @ColumnInfo(name = KEY_ACTIONS) val actions: Set, - /** - * The accessibility action id of how the user interacted - * with this node. This is null if the user didn't interact with - * this node. - */ - val userInteractedActionId: NodeInteractionType?, ) : Parcelable diff --git a/app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt b/app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt new file mode 100644 index 0000000000..6ee2b50840 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt @@ -0,0 +1,5 @@ +package io.github.sds100.keymapper.data.migration + +import androidx.room.migration.AutoMigrationSpec + +class AutoMigration18To19 : AutoMigrationSpec diff --git a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt index ee0df25555..8783d66dd5 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt @@ -1,45 +1,60 @@ package io.github.sds100.keymapper.data.repositories +import android.database.sqlite.SQLiteConstraintException +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.util.State -import io.github.sds100.keymapper.util.dataOrNull -import io.github.sds100.keymapper.util.mapData import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.update +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext interface AccessibilityNodeRepository { val nodes: Flow>> - fun get(id: Long): AccessibilityNodeEntity? - fun insert(node: AccessibilityNodeEntity) - fun deleteAll() + suspend fun get(id: Long): AccessibilityNodeEntity? + fun insert(vararg node: AccessibilityNodeEntity) + suspend fun deleteAll() } -class AccessibilityNodeRepositoryImpl(private val coroutineScope: CoroutineScope) : AccessibilityNodeRepository { - // TODO have a DAO to remember between app launches and so it isn't all cached in memory - also handles IDs automatically. - // TODO do not insert duplicates where all fields are the same - override val nodes = - MutableStateFlow>>(State.Data(ArrayList(128))) +class AccessibilityNodeRepositoryImpl( + private val coroutineScope: CoroutineScope, + private val dao: AccessibilityNodeDao, +) : AccessibilityNodeRepository { - override fun insert(node: AccessibilityNodeEntity) { - nodes.update { currentState -> - currentState.mapData { list -> - val nodeWithId = node.copy(id = list.size + 1L) - list.plus(nodeWithId) + override val nodes: StateFlow>> = + dao.getAll() + .map { list -> + // Distinct by all fields except the ID. + State.Data(list.distinctBy { it.copy(id = 0) }) + } + .flowOn(Dispatchers.IO) + .stateIn(coroutineScope, SharingStarted.WhileSubscribed(10000), State.Loading) + + override fun insert(vararg node: AccessibilityNodeEntity) { + coroutineScope.launch(Dispatchers.IO) { + for (n in node) { + try { + dao.insert(n) + } catch (e: SQLiteConstraintException) { + // Do nothing if the node already exists. + } } } } - override fun get(id: Long): AccessibilityNodeEntity? { - val nodes = nodes.value.dataOrNull() ?: return null - return nodes.find { it.id == id } + override suspend fun get(id: Long): AccessibilityNodeEntity? { + return dao.getById(id) } - override fun deleteAll() { - coroutineScope.launch { - nodes.emit(State.Data(ArrayList(128))) + override suspend fun deleteAll() { + withContext(Dispatchers.IO) { + dao.deleteAll() } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt index d8663c7153..39fea07d37 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.system.accessibility import android.os.Build import android.os.CountDownTimer import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository @@ -60,34 +61,43 @@ class AccessibilityNodeRecorder( val source = event.source ?: return + val entity = buildNodeEntity(source) ?: return + + val entities = mutableListOf() + entities.add(entity) + + if (source.childCount > 0) { + for (i in 0 until source.childCount) { + val child = source.getChild(i) ?: continue + buildNodeEntity(child)?.also { entities.add(it) } + } + } + + nodeRepository.insert(*entities.toTypedArray()) + } + + private fun buildNodeEntity(source: AccessibilityNodeInfo): AccessibilityNodeEntity? { val interactionTypes = source.actionList.mapNotNull { action -> NodeInteractionType.entries.find { it.accessibilityActionId == action.id } }.distinct() if (interactionTypes.isEmpty()) { - return + return null } - val userInteractedActionId = - NodeInteractionType.entries.find { it.accessibilityActionId == event.action } - - val entity = - AccessibilityNodeEntity( - packageName = event.packageName.toString(), - text = source.text?.toString(), - contentDescription = source.contentDescription?.toString(), - className = source.className?.toString(), - viewResourceId = source.viewIdResourceName, - uniqueId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - source.uniqueId - } else { - null - }, - actions = interactionTypes.toSet(), - userInteractedActionId = userInteractedActionId, - ) - - nodeRepository.insert(entity) + return AccessibilityNodeEntity( + packageName = source.packageName.toString(), + text = source.text?.toString(), + contentDescription = source.contentDescription?.toString(), + className = source.className?.toString(), + viewResourceId = source.viewIdResourceName, + uniqueId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + source.uniqueId + } else { + null + }, + actions = interactionTypes.toSet(), + ) } fun teardown() { From 6841c86be93306b53ac1fa64027248d299c2d485 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 20:43:47 +0200 Subject: [PATCH 15/21] #257 save all nodes within the bounds of the source event node --- .../AccessibilityNodeRecorder.kt | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt index 39fea07d37..b0e1250a56 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -1,5 +1,6 @@ package io.github.sds100.keymapper.system.accessibility +import android.graphics.Rect import android.os.Build import android.os.CountDownTimer import android.view.accessibility.AccessibilityEvent @@ -60,20 +61,47 @@ class AccessibilityNodeRecorder( } val source = event.source ?: return + val sourceBounds = Rect() + source.getBoundsInScreen(sourceBounds) - val entity = buildNodeEntity(source) ?: return + val root: AccessibilityNodeInfo = source.window.root ?: return - val entities = mutableListOf() - entities.add(entity) + // This searches for all nodes that are within the bounds of the source of the + // AccessibilityEvent because the source is not necessarily the element + // the user wants to tap. + val entities = getNodesInBounds(root, sourceBounds).toTypedArray() + nodeRepository.insert(*entities) + } + + /** + * Get all the nodes that are within the given bounds. + */ + private fun getNodesInBounds( + node: AccessibilityNodeInfo, + bounds: Rect, + ): Set { + val set = mutableSetOf() + + val nodeBounds = Rect() + node.getBoundsInScreen(nodeBounds) + + if (bounds.contains(nodeBounds)) { + val entity = buildNodeEntity(node) + + if (entity != null) { + set.add(entity) + } + } + + if (node.childCount > 0) { + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue - if (source.childCount > 0) { - for (i in 0 until source.childCount) { - val child = source.getChild(i) ?: continue - buildNodeEntity(child)?.also { entities.add(it) } + set.addAll(getNodesInBounds(child, bounds)) } } - nodeRepository.insert(*entities.toTypedArray()) + return set } private fun buildNodeEntity(source: AccessibilityNodeInfo): AccessibilityNodeEntity? { From 44b66adc63d6ad9e5759674d2258055ddfcbe4bf Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 20:45:29 +0200 Subject: [PATCH 16/21] #257 delete TODOs --- .../main/java/io/github/sds100/keymapper/backup/BackupContent.kt | 1 - .../system/accessibility/BaseAccessibilityServiceController.kt | 1 - 2 files changed, 2 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt b/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt index d9664aaba6..07a71956d1 100644 --- a/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt +++ b/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt @@ -6,7 +6,6 @@ import io.github.sds100.keymapper.data.entities.FloatingLayoutEntity import io.github.sds100.keymapper.data.entities.GroupEntity import io.github.sds100.keymapper.data.entities.KeyMapEntity -// TODO back up groups that are referenced by key maps - back up all the children as well. If the parent is not included in the back up then set the parent uid to null data class BackupContent( @SerializedName(NAME_DB_VERSION) val dbVersion: Int, diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt index d2206d813e..d0a3347b75 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt @@ -573,7 +573,6 @@ abstract class BaseAccessibilityServiceController( } private fun recordTriggerJob() = coroutineScope.launch { - // TODO use Kotlin timer. repeat(RECORD_TRIGGER_TIMER_LENGTH) { iteration -> if (isActive) { val timeLeft = RECORD_TRIGGER_TIMER_LENGTH - iteration From b96fca3eb170880dcbf112b215de5a0185abf9a4 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 20:50:04 +0200 Subject: [PATCH 17/21] #257 add to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e501ffdd2..c585e6f557 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Added - #699 Time constraints ⏰ +- #257 Action to interact with user interface elements inside other apps. ## Changed From 8c8e0206dcbb3ba9c9cc6252e0b460b940936d3e Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 21:03:23 +0200 Subject: [PATCH 18/21] #257 fix style and free build compilation issues --- .../system/accessibility/AccessibilityServiceController.kt | 5 ++++- .../sds100/keymapper/actions/ActionDataEntityMapper.kt | 2 +- .../sds100/keymapper/actions/FlashlightActionBottomSheet.kt | 2 -- .../io/github/sds100/keymapper/constraints/ConstraintId.kt | 2 +- .../sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt | 2 +- .../keymapper/data/entities/AccessibilityNodeEntity.kt | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt b/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt index 3d2a18d9cf..8e0fb9dda5 100644 --- a/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt +++ b/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt @@ -2,6 +2,7 @@ package io.github.sds100.keymapper.system.accessibility import io.github.sds100.keymapper.actions.PerformActionsUseCase import io.github.sds100.keymapper.constraints.DetectConstraintsUseCase +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.mappings.FingerprintGesturesSupportedUseCase import io.github.sds100.keymapper.mappings.PauseKeyMapsUseCase @@ -17,7 +18,7 @@ import kotlinx.coroutines.flow.SharedFlow class AccessibilityServiceController( coroutineScope: CoroutineScope, - accessibilityService: IAccessibilityService, + accessibilityService: MyAccessibilityService, inputEvents: SharedFlow, outputEvents: MutableSharedFlow, detectConstraintsUseCase: DetectConstraintsUseCase, @@ -30,6 +31,7 @@ class AccessibilityServiceController( suAdapter: SuAdapter, inputMethodAdapter: InputMethodAdapter, settingsRepository: PreferenceRepository, + nodeRepository: AccessibilityNodeRepository, ) : BaseAccessibilityServiceController( coroutineScope, accessibilityService, @@ -45,4 +47,5 @@ class AccessibilityServiceController( suAdapter, inputMethodAdapter, settingsRepository, + nodeRepository, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index b6b99e1726..ad26f41cdd 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -329,7 +329,7 @@ object ActionDataEntityMapper { ActionId.TOGGLE_DND_MODE, ActionId.ENABLE_DND_MODE, - -> { + -> { val dndMode = entity.extras.getData(ActionEntity.EXTRA_DND_MODE).then { DND_MODE_MAP.getKey(it)!!.success() }.valueOrNull() ?: return null diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt b/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt index 346dc1114b..1482dd5327 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt @@ -35,9 +35,7 @@ import androidx.compose.material3.SheetValue import androidx.compose.material3.Slider import androidx.compose.material3.Text import androidx.compose.material3.TextButton -import androidx.compose.material3.TimePicker import androidx.compose.material3.rememberModalBottomSheetState -import androidx.compose.material3.rememberTimePickerState import androidx.compose.runtime.Composable import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope diff --git a/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt b/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt index 0c7f2678ae..b015752aca 100644 --- a/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt +++ b/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt @@ -49,5 +49,5 @@ enum class ConstraintId { CHARGING, DISCHARGING, - TIME + TIME, } diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt index b86cf032bd..ed2bd36250 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt @@ -32,4 +32,4 @@ interface AccessibilityNodeDao { @Query("DELETE FROM $TABLE_NAME") suspend fun deleteAll() -} +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt index 896a12e8a5..ca3ba0ee1c 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -19,7 +19,7 @@ import kotlinx.serialization.Serializable @Serializable @Parcelize -@Entity(tableName = TABLE_NAME,) +@Entity(tableName = TABLE_NAME) data class AccessibilityNodeEntity( @PrimaryKey(autoGenerate = true) @ColumnInfo(name = KEY_ID) From 00480dd4bf03d8fcaefde84e4d1695743491c2dc Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 21:18:22 +0200 Subject: [PATCH 19/21] #257 fix tests and style --- .../sds100/keymapper/actions/ActionDataEntityMapper.kt | 2 +- .../java/io/github/sds100/keymapper/backup/BackupManager.kt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index ad26f41cdd..4133b18d4d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -356,7 +356,7 @@ object ActionDataEntityMapper { ActionId.PREVIOUS_TRACK_PACKAGE, ActionId.FAST_FORWARD_PACKAGE, ActionId.REWIND_PACKAGE, - -> { + -> { val packageName = entity.extras.getData(ActionEntity.EXTRA_PACKAGE_NAME).valueOrNull() ?: return null diff --git a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt index 659c66b79e..5f0667e8f4 100644 --- a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt +++ b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt @@ -225,7 +225,7 @@ class BackupManagerImpl( val deviceInfoList by rootElement.byNullableArray(BackupContent.NAME_DEVICE_INFO) val migratedKeyMapList = mutableListOf() - + ac val keyMapMigrations = listOf( JsonMigration(9, 10) { json -> Migration9To10.migrateJson(json) }, JsonMigration(10, 11) { json -> Migration10To11.migrateJson(json) }, @@ -249,6 +249,9 @@ class BackupManagerImpl( // Do nothing. It just removed the group name index. JsonMigration(17, 18) { json -> json }, + + // Do nothing. Just added the accessibility node table. + JsonMigration(18, 19) { json -> json }, ) if (keyMapListJsonArray != null) { From 85bba021d9e5cc9d3f60f933a30cbddf2750117c Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 21:49:07 +0200 Subject: [PATCH 20/21] #257 delete random characters --- .../java/io/github/sds100/keymapper/backup/BackupManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt index 5f0667e8f4..64bf0f770c 100644 --- a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt +++ b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt @@ -225,7 +225,7 @@ class BackupManagerImpl( val deviceInfoList by rootElement.byNullableArray(BackupContent.NAME_DEVICE_INFO) val migratedKeyMapList = mutableListOf() - ac + val keyMapMigrations = listOf( JsonMigration(9, 10) { json -> Migration9To10.migrateJson(json) }, JsonMigration(10, 11) { json -> Migration10To11.migrateJson(json) }, From 38b3f09d0d9a7c4dd5975e538795c24cf4050ab0 Mon Sep 17 00:00:00 2001 From: sds100 Date: Sun, 4 May 2025 22:47:19 +0200 Subject: [PATCH 21/21] #257 fix style --- .../java/io/github/sds100/keymapper/backup/BackupManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt index 64bf0f770c..f2e22b2ce8 100644 --- a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt +++ b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt @@ -225,7 +225,7 @@ class BackupManagerImpl( val deviceInfoList by rootElement.byNullableArray(BackupContent.NAME_DEVICE_INFO) val migratedKeyMapList = mutableListOf() - + val keyMapMigrations = listOf( JsonMigration(9, 10) { json -> Migration9To10.migrateJson(json) }, JsonMigration(10, 11) { json -> Migration10To11.migrateJson(json) },