diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b43105a25..c585e6f557 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ ## Added - #699 Time constraints ⏰ +- #257 Action to interact with user interface elements inside other apps. + +## Changed + +- Rename tap screen actions inside key maps. ## [3.0.1](https://github.com/sds100/KeyMapper/releases/tag/v3.0.1) diff --git a/app/build.gradle b/app/build.gradle index c7ed7dc671..e442585f81 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -165,7 +165,7 @@ dependencies { compileOnly project(":systemstubs") - def room_version = "2.6.1" + def room_version = "2.7.1" def coroutinesVersion = "1.9.0" def nav_version = '2.8.9' def epoxy_version = "4.6.2" @@ -179,7 +179,7 @@ dependencies { implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.8.0" // random stuff - implementation "com.google.android.material:material:1.13.0-alpha12" + implementation "com.google.android.material:material:1.13.0-alpha13" implementation "com.github.salomonbrys.kotson:kotson:2.5.0" implementation "com.airbnb.android:epoxy:$epoxy_version" implementation "com.airbnb.android:epoxy-databinding:$epoxy_version" @@ -192,7 +192,7 @@ dependencies { implementation "dev.rikka.shizuku:api:$shizuku_version" implementation "dev.rikka.shizuku:provider:$shizuku_version" implementation "org.lsposed.hiddenapibypass:hiddenapibypass:4.3" - proImplementation 'com.revenuecat.purchases:purchases:8.15.0' + proImplementation 'com.revenuecat.purchases:purchases:8.17.0' proImplementation "com.airbnb.android:lottie-compose:6.6.3" implementation("com.squareup.okhttp3:okhttp:4.12.0") coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") @@ -207,7 +207,7 @@ dependencies { // androidx implementation "androidx.legacy:legacy-support-core-ui:1.0.0" - implementation "androidx.core:core-ktx:1.15.0" + implementation "androidx.core:core-ktx:1.16.0" implementation "androidx.activity:activity-ktx:1.10.1" implementation "androidx.fragment:fragment-ktx:1.8.6" @@ -233,7 +233,7 @@ dependencies { ksp "androidx.room:room-compiler:$room_version" // Compose - Dependency composeBom = platform('androidx.compose:compose-bom-beta:2025.03.01') + Dependency composeBom = platform('androidx.compose:compose-bom-beta:2025.04.01') implementation composeBom implementation 'androidx.compose.foundation:foundation' implementation "androidx.compose.ui:ui-android" diff --git a/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json b/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json new file mode 100644 index 0000000000..ed9aef420f --- /dev/null +++ b/app/schemas/io.github.sds100.keymapper.data.db.AppDatabase/19.json @@ -0,0 +1,441 @@ +{ + "formatVersion": 1, + "database": { + "version": 19, + "identityHash": "300aa53acf7905efdf0c5b0ff7516ec9", + "entities": [ + { + "tableName": "keymaps", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `trigger` TEXT NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `flags` INTEGER NOT NULL, `is_enabled` INTEGER NOT NULL, `uid` TEXT NOT NULL, `group_uid` TEXT, FOREIGN KEY(`group_uid`) REFERENCES `groups`(`uid`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "trigger", + "columnName": "trigger", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "actionList", + "columnName": "action_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraint_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isEnabled", + "columnName": "is_enabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "groupUid", + "columnName": "group_uid", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_keymaps_uid", + "unique": true, + "columnNames": [ + "uid" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_keymaps_uid` ON `${TABLE_NAME}` (`uid`)" + } + ], + "foreignKeys": [ + { + "table": "groups", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "group_uid" + ], + "referencedColumns": [ + "uid" + ] + } + ] + }, + { + "tableName": "fingerprintmaps", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `action_list` TEXT NOT NULL, `constraint_list` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `extras` TEXT NOT NULL, `flags` INTEGER NOT NULL, `is_enabled` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionList", + "columnName": "action_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraint_list", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "extras", + "columnName": "extras", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "flags", + "columnName": "flags", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isEnabled", + "columnName": "is_enabled", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `time` INTEGER NOT NULL, `severity` INTEGER NOT NULL, `message` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "time", + "columnName": "time", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "severity", + "columnName": "severity", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "message", + "columnName": "message", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "floating_layouts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` TEXT NOT NULL, `name` TEXT NOT NULL, PRIMARY KEY(`uid`))", + "fields": [ + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uid" + ] + }, + "indices": [ + { + "name": "index_floating_layouts_name", + "unique": true, + "columnNames": [ + "name" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_floating_layouts_name` ON `${TABLE_NAME}` (`name`)" + } + ] + }, + { + "tableName": "floating_buttons", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` TEXT NOT NULL, `layout_uid` TEXT NOT NULL, `text` TEXT NOT NULL, `button_size` INTEGER NOT NULL, `x` INTEGER NOT NULL, `y` INTEGER NOT NULL, `orientation` TEXT NOT NULL, `display_width` INTEGER NOT NULL, `display_height` INTEGER NOT NULL, `border_opacity` REAL, `background_opacity` REAL, PRIMARY KEY(`uid`), FOREIGN KEY(`layout_uid`) REFERENCES `floating_layouts`(`uid`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "layoutUid", + "columnName": "layout_uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "buttonSize", + "columnName": "button_size", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "x", + "columnName": "x", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "y", + "columnName": "y", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "orientation", + "columnName": "orientation", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayWidth", + "columnName": "display_width", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "displayHeight", + "columnName": "display_height", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "borderOpacity", + "columnName": "border_opacity", + "affinity": "REAL" + }, + { + "fieldPath": "backgroundOpacity", + "columnName": "background_opacity", + "affinity": "REAL" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uid" + ] + }, + "indices": [ + { + "name": "index_floating_buttons_layout_uid", + "unique": false, + "columnNames": [ + "layout_uid" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_floating_buttons_layout_uid` ON `${TABLE_NAME}` (`layout_uid`)" + } + ], + "foreignKeys": [ + { + "table": "floating_layouts", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "layout_uid" + ], + "referencedColumns": [ + "uid" + ] + } + ] + }, + { + "tableName": "groups", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` TEXT NOT NULL, `name` TEXT NOT NULL, `constraints` TEXT NOT NULL, `constraint_mode` INTEGER NOT NULL, `parent_uid` TEXT, `last_opened_date` INTEGER, PRIMARY KEY(`uid`), FOREIGN KEY(`parent_uid`) REFERENCES `groups`(`uid`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintList", + "columnName": "constraints", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "constraintMode", + "columnName": "constraint_mode", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "parentUid", + "columnName": "parent_uid", + "affinity": "TEXT" + }, + { + "fieldPath": "lastOpenedDate", + "columnName": "last_opened_date", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "uid" + ] + }, + "foreignKeys": [ + { + "table": "groups", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "parent_uid" + ], + "referencedColumns": [ + "uid" + ] + } + ] + }, + { + "tableName": "accessibility_nodes", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `package_name` TEXT NOT NULL, `text` TEXT, `content_description` TEXT, `class_name` TEXT, `view_resource_id` TEXT, `unique_id` TEXT, `actions` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "packageName", + "columnName": "package_name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "text", + "columnName": "text", + "affinity": "TEXT" + }, + { + "fieldPath": "contentDescription", + "columnName": "content_description", + "affinity": "TEXT" + }, + { + "fieldPath": "className", + "columnName": "class_name", + "affinity": "TEXT" + }, + { + "fieldPath": "viewResourceId", + "columnName": "view_resource_id", + "affinity": "TEXT" + }, + { + "fieldPath": "uniqueId", + "columnName": "unique_id", + "affinity": "TEXT" + }, + { + "fieldPath": "actions", + "columnName": "actions", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '300aa53acf7905efdf0c5b0ff7516ec9')" + ] + } +} \ No newline at end of file diff --git a/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt b/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt index 3d2a18d9cf..8e0fb9dda5 100644 --- a/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt +++ b/app/src/free/java/io/github/sds100/keymapper/system/accessibility/AccessibilityServiceController.kt @@ -2,6 +2,7 @@ package io.github.sds100.keymapper.system.accessibility import io.github.sds100.keymapper.actions.PerformActionsUseCase import io.github.sds100.keymapper.constraints.DetectConstraintsUseCase +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.mappings.FingerprintGesturesSupportedUseCase import io.github.sds100.keymapper.mappings.PauseKeyMapsUseCase @@ -17,7 +18,7 @@ import kotlinx.coroutines.flow.SharedFlow class AccessibilityServiceController( coroutineScope: CoroutineScope, - accessibilityService: IAccessibilityService, + accessibilityService: MyAccessibilityService, inputEvents: SharedFlow, outputEvents: MutableSharedFlow, detectConstraintsUseCase: DetectConstraintsUseCase, @@ -30,6 +31,7 @@ class AccessibilityServiceController( suAdapter: SuAdapter, inputMethodAdapter: InputMethodAdapter, settingsRepository: PreferenceRepository, + nodeRepository: AccessibilityNodeRepository, ) : BaseAccessibilityServiceController( coroutineScope, accessibilityService, @@ -45,4 +47,5 @@ class AccessibilityServiceController( suAdapter, inputMethodAdapter, settingsRepository, + nodeRepository, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt b/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt index b5b3275dc6..162bbd5858 100644 --- a/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt +++ b/app/src/main/java/io/github/sds100/keymapper/KeyMapperApp.kt @@ -12,6 +12,7 @@ import androidx.lifecycle.LifecycleObserver import androidx.lifecycle.OnLifecycleEvent import androidx.lifecycle.ProcessLifecycleOwner import androidx.multidex.MultiDexApplication +import io.github.sds100.keymapper.actions.uielement.InteractUiElementController import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.entities.LogEntryEntity import io.github.sds100.keymapper.logging.KeyMapperLoggingTree @@ -151,6 +152,15 @@ class KeyMapperApp : MultiDexApplication() { RecordTriggerController(appCoroutineScope, accessibilityServiceAdapter) } + val interactUiElementController by lazy { + InteractUiElementController( + appCoroutineScope, + accessibilityServiceAdapter, + ServiceLocator.accessibilityNodeRepository(this), + packageManagerAdapter, + ) + } + val autoGrantPermissionController by lazy { AutoGrantPermissionController( appCoroutineScope, diff --git a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt index 00f14520f2..c0dd401af4 100755 --- a/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt +++ b/app/src/main/java/io/github/sds100/keymapper/ServiceLocator.kt @@ -8,6 +8,8 @@ import io.github.sds100.keymapper.actions.sound.SoundsManagerImpl import io.github.sds100.keymapper.backup.BackupManager import io.github.sds100.keymapper.backup.BackupManagerImpl import io.github.sds100.keymapper.data.db.AppDatabase +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepositoryImpl import io.github.sds100.keymapper.data.repositories.FloatingButtonRepository import io.github.sds100.keymapper.data.repositories.FloatingLayoutRepository import io.github.sds100.keymapper.data.repositories.GroupRepository @@ -214,6 +216,20 @@ object ServiceLocator { ) } + @Volatile + private var accessibilityNodeRepository: AccessibilityNodeRepository? = null + + fun accessibilityNodeRepository(context: Context): AccessibilityNodeRepository { + synchronized(this) { + return accessibilityNodeRepository ?: AccessibilityNodeRepositoryImpl( + (context.applicationContext as KeyMapperApp).appCoroutineScope, + database(context).accessibilityNodeDao(), + ).also { + this.accessibilityNodeRepository = it + } + } + } + fun fileAdapter(context: Context): FileAdapter = (context.applicationContext as KeyMapperApp).fileAdapter fun inputMethodAdapter(context: Context): InputMethodAdapter = (context.applicationContext as KeyMapperApp).inputMethodAdapter diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt index 5f4799294d..3823a2e5f7 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionData.kt @@ -1,6 +1,7 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.system.camera.CameraLens import io.github.sds100.keymapper.system.display.Orientation import io.github.sds100.keymapper.system.intents.IntentExtraModel @@ -260,7 +261,7 @@ sealed class ActionData : Comparable { } @Serializable - object Disable : DoNotDisturb() { + data object Disable : DoNotDisturb() { override val id = ActionId.DISABLE_DND_MODE } } @@ -268,32 +269,32 @@ sealed class ActionData : Comparable { @Serializable sealed class Rotation : ActionData() { @Serializable - object EnableAuto : Rotation() { + data object EnableAuto : Rotation() { override val id = ActionId.ENABLE_AUTO_ROTATE } @Serializable - object DisableAuto : Rotation() { + data object DisableAuto : Rotation() { override val id = ActionId.DISABLE_AUTO_ROTATE } @Serializable - object ToggleAuto : Rotation() { + data object ToggleAuto : Rotation() { override val id = ActionId.TOGGLE_AUTO_ROTATE } @Serializable - object Portrait : Rotation() { + data object Portrait : Rotation() { override val id = ActionId.PORTRAIT_MODE } @Serializable - object Landscape : Rotation() { + data object Landscape : Rotation() { override val id = ActionId.LANDSCAPE_MODE } @Serializable - object SwitchOrientation : Rotation() { + data object SwitchOrientation : Rotation() { override val id = ActionId.SWITCH_ORIENTATION } @@ -369,37 +370,37 @@ sealed class ActionData : Comparable { @Serializable sealed class ControlMedia : ActionData() { @Serializable - object Pause : ControlMedia() { + data object Pause : ControlMedia() { override val id = ActionId.PAUSE_MEDIA } @Serializable - object Play : ControlMedia() { + data object Play : ControlMedia() { override val id = ActionId.PLAY_MEDIA } @Serializable - object PlayPause : ControlMedia() { + data object PlayPause : ControlMedia() { override val id = ActionId.PLAY_PAUSE_MEDIA } @Serializable - object NextTrack : ControlMedia() { + data object NextTrack : ControlMedia() { override val id = ActionId.NEXT_TRACK } @Serializable - object PreviousTrack : ControlMedia() { + data object PreviousTrack : ControlMedia() { override val id = ActionId.PREVIOUS_TRACK } @Serializable - object FastForward : ControlMedia() { + data object FastForward : ControlMedia() { override val id = ActionId.FAST_FORWARD } @Serializable - object Rewind : ControlMedia() { + data object Rewind : ControlMedia() { override val id = ActionId.REWIND } } @@ -537,17 +538,17 @@ sealed class ActionData : Comparable { @Serializable sealed class Wifi : ActionData() { @Serializable - object Enable : Wifi() { + data object Enable : Wifi() { override val id = ActionId.ENABLE_WIFI } @Serializable - object Disable : Wifi() { + data object Disable : Wifi() { override val id = ActionId.DISABLE_WIFI } @Serializable - object Toggle : Wifi() { + data object Toggle : Wifi() { override val id = ActionId.TOGGLE_WIFI } } @@ -555,17 +556,17 @@ sealed class ActionData : Comparable { @Serializable sealed class Bluetooth : ActionData() { @Serializable - object Enable : Bluetooth() { + data object Enable : Bluetooth() { override val id = ActionId.ENABLE_BLUETOOTH } @Serializable - object Disable : Bluetooth() { + data object Disable : Bluetooth() { override val id = ActionId.DISABLE_BLUETOOTH } @Serializable - object Toggle : Bluetooth() { + data object Toggle : Bluetooth() { override val id = ActionId.TOGGLE_BLUETOOTH } } @@ -573,17 +574,17 @@ sealed class ActionData : Comparable { @Serializable sealed class Nfc : ActionData() { @Serializable - object Enable : Nfc() { + data object Enable : Nfc() { override val id = ActionId.ENABLE_NFC } @Serializable - object Disable : Nfc() { + data object Disable : Nfc() { override val id = ActionId.DISABLE_NFC } @Serializable - object Toggle : Nfc() { + data object Toggle : Nfc() { override val id = ActionId.TOGGLE_NFC } } @@ -591,17 +592,17 @@ sealed class ActionData : Comparable { @Serializable sealed class AirplaneMode : ActionData() { @Serializable - object Enable : AirplaneMode() { + data object Enable : AirplaneMode() { override val id = ActionId.ENABLE_AIRPLANE_MODE } @Serializable - object Disable : AirplaneMode() { + data object Disable : AirplaneMode() { override val id = ActionId.DISABLE_AIRPLANE_MODE } @Serializable - object Toggle : AirplaneMode() { + data object Toggle : AirplaneMode() { override val id = ActionId.TOGGLE_AIRPLANE_MODE } } @@ -609,17 +610,17 @@ sealed class ActionData : Comparable { @Serializable sealed class MobileData : ActionData() { @Serializable - object Enable : MobileData() { + data object Enable : MobileData() { override val id = ActionId.ENABLE_MOBILE_DATA } @Serializable - object Disable : MobileData() { + data object Disable : MobileData() { override val id = ActionId.DISABLE_MOBILE_DATA } @Serializable - object Toggle : MobileData() { + data object Toggle : MobileData() { override val id = ActionId.TOGGLE_MOBILE_DATA } } @@ -627,27 +628,27 @@ sealed class ActionData : Comparable { @Serializable sealed class Brightness : ActionData() { @Serializable - object EnableAuto : Brightness() { + data object EnableAuto : Brightness() { override val id = ActionId.ENABLE_AUTO_BRIGHTNESS } @Serializable - object DisableAuto : Brightness() { + data object DisableAuto : Brightness() { override val id = ActionId.DISABLE_AUTO_BRIGHTNESS } @Serializable - object ToggleAuto : Brightness() { + data object ToggleAuto : Brightness() { override val id = ActionId.TOGGLE_AUTO_BRIGHTNESS } @Serializable - object Increase : Brightness() { + data object Increase : Brightness() { override val id = ActionId.INCREASE_BRIGHTNESS } @Serializable - object Decrease : Brightness() { + data object Decrease : Brightness() { override val id = ActionId.DECREASE_BRIGHTNESS } } @@ -655,178 +656,178 @@ sealed class ActionData : Comparable { @Serializable sealed class StatusBar : ActionData() { @Serializable - object ExpandNotifications : StatusBar() { + data object ExpandNotifications : StatusBar() { override val id = ActionId.EXPAND_NOTIFICATION_DRAWER } @Serializable - object ToggleNotifications : StatusBar() { + data object ToggleNotifications : StatusBar() { override val id = ActionId.TOGGLE_NOTIFICATION_DRAWER } @Serializable - object ExpandQuickSettings : StatusBar() { + data object ExpandQuickSettings : StatusBar() { override val id = ActionId.EXPAND_QUICK_SETTINGS } @Serializable - object ToggleQuickSettings : StatusBar() { + data object ToggleQuickSettings : StatusBar() { override val id = ActionId.TOGGLE_QUICK_SETTINGS } @Serializable - object Collapse : StatusBar() { + data object Collapse : StatusBar() { override val id = ActionId.COLLAPSE_STATUS_BAR } } @Serializable - object GoBack : ActionData() { + data object GoBack : ActionData() { override val id = ActionId.GO_BACK } @Serializable - object GoHome : ActionData() { + data object GoHome : ActionData() { override val id = ActionId.GO_HOME } @Serializable - object OpenRecents : ActionData() { + data object OpenRecents : ActionData() { override val id = ActionId.OPEN_RECENTS } @Serializable - object GoLastApp : ActionData() { + data object GoLastApp : ActionData() { override val id = ActionId.GO_LAST_APP } @Serializable - object OpenMenu : ActionData() { + data object OpenMenu : ActionData() { override val id = ActionId.OPEN_MENU } @Serializable - object ToggleSplitScreen : ActionData() { + data object ToggleSplitScreen : ActionData() { override val id = ActionId.TOGGLE_SPLIT_SCREEN } @Serializable - object Screenshot : ActionData() { + data object Screenshot : ActionData() { override val id = ActionId.SCREENSHOT } @Serializable - object MoveCursorToEnd : ActionData() { + data object MoveCursorToEnd : ActionData() { override val id = ActionId.MOVE_CURSOR_TO_END } @Serializable - object ToggleKeyboard : ActionData() { + data object ToggleKeyboard : ActionData() { override val id = ActionId.TOGGLE_KEYBOARD } @Serializable - object ShowKeyboard : ActionData() { + data object ShowKeyboard : ActionData() { override val id = ActionId.SHOW_KEYBOARD } @Serializable - object HideKeyboard : ActionData() { + data object HideKeyboard : ActionData() { override val id = ActionId.HIDE_KEYBOARD } @Serializable - object ShowKeyboardPicker : ActionData() { + data object ShowKeyboardPicker : ActionData() { override val id = ActionId.SHOW_KEYBOARD_PICKER } @Serializable - object CopyText : ActionData() { + data object CopyText : ActionData() { override val id = ActionId.TEXT_COPY } @Serializable - object PasteText : ActionData() { + data object PasteText : ActionData() { override val id = ActionId.TEXT_PASTE } @Serializable - object CutText : ActionData() { + data object CutText : ActionData() { override val id = ActionId.TEXT_CUT } @Serializable - object SelectWordAtCursor : ActionData() { + data object SelectWordAtCursor : ActionData() { override val id = ActionId.SELECT_WORD_AT_CURSOR } @Serializable - object VoiceAssistant : ActionData() { + data object VoiceAssistant : ActionData() { override val id = ActionId.OPEN_VOICE_ASSISTANT } @Serializable - object DeviceAssistant : ActionData() { + data object DeviceAssistant : ActionData() { override val id = ActionId.OPEN_DEVICE_ASSISTANT } @Serializable - object OpenCamera : ActionData() { + data object OpenCamera : ActionData() { override val id = ActionId.OPEN_CAMERA } @Serializable - object LockDevice : ActionData() { + data object LockDevice : ActionData() { override val id = ActionId.LOCK_DEVICE } @Serializable - object ScreenOnOff : ActionData() { + data object ScreenOnOff : ActionData() { override val id = ActionId.POWER_ON_OFF_DEVICE } @Serializable - object SecureLock : ActionData() { + data object SecureLock : ActionData() { override val id = ActionId.SECURE_LOCK_DEVICE } @Serializable - object ConsumeKeyEvent : ActionData() { + data object ConsumeKeyEvent : ActionData() { override val id = ActionId.CONSUME_KEY_EVENT } @Serializable - object OpenSettings : ActionData() { + data object OpenSettings : ActionData() { override val id = ActionId.OPEN_SETTINGS } @Serializable - object ShowPowerMenu : ActionData() { + data object ShowPowerMenu : ActionData() { override val id = ActionId.SHOW_POWER_MENU } @Serializable - object DismissLastNotification : ActionData() { + data object DismissLastNotification : ActionData() { override val id: ActionId = ActionId.DISMISS_MOST_RECENT_NOTIFICATION } @Serializable - object DismissAllNotifications : ActionData() { + data object DismissAllNotifications : ActionData() { override val id: ActionId = ActionId.DISMISS_ALL_NOTIFICATIONS } @Serializable - object AnswerCall : ActionData() { + data object AnswerCall : ActionData() { override val id: ActionId = ActionId.ANSWER_PHONE_CALL } @Serializable - object EndCall : ActionData() { + data object EndCall : ActionData() { override val id: ActionId = ActionId.END_PHONE_CALL } @Serializable - object DeviceControls : ActionData() { + data object DeviceControls : ActionData() { override val id: ActionId = ActionId.DEVICE_CONTROLS } @@ -845,4 +846,22 @@ sealed class ActionData : Comparable { return "HttpRequest(description=$description)" } } + + @Serializable + data class InteractUiElement( + val description: String, + val nodeAction: NodeInteractionType, + val packageName: String, + val text: String?, + val contentDescription: String?, + val className: String?, + val viewResourceId: String?, + val uniqueId: String?, + /** + * A list of the allowed accessibility node actions. + */ + val nodeActions: Set, + ) : ActionData() { + override val id: ActionId = ActionId.INTERACT_UI_ELEMENT + } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt index 53a6515254..4133b18d4d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionDataEntityMapper.kt @@ -1,7 +1,9 @@ package io.github.sds100.keymapper.actions import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType import io.github.sds100.keymapper.data.db.typeconverter.ConstantTypeConverters +import io.github.sds100.keymapper.data.db.typeconverter.NodeInteractionTypeSetTypeConverter import io.github.sds100.keymapper.data.entities.ActionEntity import io.github.sds100.keymapper.data.entities.EntityExtra import io.github.sds100.keymapper.data.entities.getData @@ -12,6 +14,9 @@ import io.github.sds100.keymapper.system.network.HttpMethod import io.github.sds100.keymapper.system.volume.DndMode import io.github.sds100.keymapper.system.volume.RingerMode import io.github.sds100.keymapper.system.volume.VolumeStream +import io.github.sds100.keymapper.util.Error +import io.github.sds100.keymapper.util.Result +import io.github.sds100.keymapper.util.Success import io.github.sds100.keymapper.util.getKey import io.github.sds100.keymapper.util.success import io.github.sds100.keymapper.util.then @@ -41,6 +46,8 @@ object ActionDataEntityMapper { ActionEntity.Type.SYSTEM_ACTION -> { SYSTEM_ACTION_ID_MAP.getKey(entity.data) ?: return null } + + ActionEntity.Type.INTERACT_UI_ELEMENT -> ActionId.INTERACT_UI_ELEMENT } return when (actionId) { @@ -522,9 +529,59 @@ object ActionDataEntityMapper { authorizationHeader = authorizationHeader, ) } + + ActionId.INTERACT_UI_ELEMENT -> { + val packageName = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME) + .valueOrNull()!! + + val contentDescription = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION) + .valueOrNull() + + val text = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_TEXT).valueOrNull() + + val className = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_CLASS_NAME).valueOrNull() + + val viewResourceId = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID) + .valueOrNull() + + val uniqueId = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID).valueOrNull() + + val actions = entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS).then { + Success(NodeInteractionTypeSetTypeConverter().toSet(it.toInt())) + }.valueOrNull() ?: emptySet() + + val nodeAction = + entity.extras.getData(ActionEntity.EXTRA_ACCESSIBILITY_NODE_ACTION).then { + convertNodeInteractionType(it) + }.valueOrNull() ?: return null + + ActionData.InteractUiElement( + description = entity.data, + nodeAction = nodeAction, + packageName = packageName, + text = text, + contentDescription = contentDescription, + className = className, + viewResourceId = viewResourceId, + uniqueId = uniqueId, + nodeActions = actions, + ) + } } } + private fun convertNodeInteractionType(string: String): Result = try { + Success(NodeInteractionType.valueOf(string)) + } catch (e: IllegalArgumentException) { + Error.Exception(e) + } + fun toEntity(data: ActionData): ActionEntity { val type = when (data) { is ActionData.Intent -> ActionEntity.Type.INTENT @@ -538,6 +595,7 @@ object ActionDataEntityMapper { is ActionData.Text -> ActionEntity.Type.TEXT_BLOCK is ActionData.Url -> ActionEntity.Type.URL is ActionData.Sound -> ActionEntity.Type.SOUND + is ActionData.InteractUiElement -> ActionEntity.Type.INTERACT_UI_ELEMENT else -> ActionEntity.Type.SYSTEM_ACTION } @@ -579,6 +637,7 @@ object ActionDataEntityMapper { is ActionData.Text -> data.text is ActionData.Url -> data.url is ActionData.Sound -> data.soundUid + is ActionData.InteractUiElement -> data.description else -> SYSTEM_ACTION_ID_MAP[data.id]!! } @@ -750,6 +809,69 @@ object ActionDataEntityMapper { ), ) + is ActionData.InteractUiElement -> buildList { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_NODE_ACTION, + data.nodeAction.toString(), + ), + ) + + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_PACKAGE_NAME, + data.packageName, + ), + ) + + data.contentDescription?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION, + it, + ), + ) + } + + data.text?.let { add(EntityExtra(ActionEntity.EXTRA_ACCESSIBILITY_TEXT, it)) } + + data.className?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_CLASS_NAME, + it, + ), + ) + } + + data.viewResourceId?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID, + it, + ), + ) + } + + data.uniqueId?.let { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_UNIQUE_ID, + it, + ), + ) + } + + if (data.nodeActions.isNotEmpty()) { + add( + EntityExtra( + ActionEntity.EXTRA_ACCESSIBILITY_ACTIONS, + NodeInteractionTypeSetTypeConverter().toMask(data.nodeActions).toString(), + ), + ) + } + } + else -> emptyList() } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt index 515474d744..011c7f825a 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionId.kt @@ -16,6 +16,7 @@ enum class ActionId { INTENT, PHONE_CALL, SOUND, + INTERACT_UI_ELEMENT, TOGGLE_WIFI, ENABLE_WIFI, diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt index 755353b04d..ccd1fac039 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUiHelper.kt @@ -356,7 +356,7 @@ class ActionUiHelper( } else { getString( R.string.description_tap_coordinate_with_description, - arrayOf(action.x, action.y, action.description), + action.description, ) } @@ -529,6 +529,8 @@ class ActionUiHelper( ActionData.DeviceControls -> getString(R.string.action_device_controls) is ActionData.HttpRequest -> action.description + + is ActionData.InteractUiElement -> action.description } fun getIcon(action: ActionData): ComposeIconInfo = when (action) { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt index e3b696fc8c..a2a0d82c73 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ActionUtils.kt @@ -73,6 +73,7 @@ import io.github.sds100.keymapper.R import io.github.sds100.keymapper.system.permissions.Permission import io.github.sds100.keymapper.util.ui.compose.icons.HomeIotDevice import io.github.sds100.keymapper.util.ui.compose.icons.InstantMix +import io.github.sds100.keymapper.util.ui.compose.icons.JumpToElement import io.github.sds100.keymapper.util.ui.compose.icons.KeyMapperIcons import io.github.sds100.keymapper.util.ui.compose.icons.MatchWord import io.github.sds100.keymapper.util.ui.compose.icons.NfcOff @@ -230,6 +231,8 @@ object ActionUtils { ActionId.DISMISS_MOST_RECENT_NOTIFICATION -> ActionCategory.NOTIFICATIONS ActionId.DISMISS_ALL_NOTIFICATIONS -> ActionCategory.NOTIFICATIONS ActionId.DEVICE_CONTROLS -> ActionCategory.APPS + + ActionId.INTERACT_UI_ELEMENT -> ActionCategory.APPS } @StringRes @@ -342,6 +345,7 @@ object ActionUtils { ActionId.END_PHONE_CALL -> R.string.action_end_call ActionId.DEVICE_CONTROLS -> R.string.action_device_controls ActionId.HTTP_REQUEST -> R.string.action_http_request + ActionId.INTERACT_UI_ELEMENT -> R.string.action_interact_ui_element_title } @DrawableRes @@ -454,6 +458,7 @@ object ActionUtils { ActionId.END_PHONE_CALL -> R.drawable.ic_outline_call_end_24 ActionId.DEVICE_CONTROLS -> R.drawable.ic_home_automation ActionId.HTTP_REQUEST -> null + ActionId.INTERACT_UI_ELEMENT -> null } fun getMinApi(id: ActionId): Int = when (id) { @@ -770,6 +775,7 @@ object ActionUtils { ActionId.END_PHONE_CALL -> Icons.Outlined.CallEnd ActionId.DEVICE_CONTROLS -> KeyMapperIcons.HomeIotDevice ActionId.HTTP_REQUEST -> Icons.Outlined.Http + ActionId.INTERACT_UI_ELEMENT -> KeyMapperIcons.JumpToElement } } @@ -821,6 +827,7 @@ fun ActionData.isEditable(): Boolean = when (this) { is ActionData.Url, is ActionData.PhoneCall, is ActionData.HttpRequest, + is ActionData.InteractUiElement, -> true else -> false diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt index 9001bf183b..34cc3a8616 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ChooseActionScreen.kt @@ -15,27 +15,18 @@ import androidx.compose.foundation.lazy.grid.GridCells import androidx.compose.foundation.lazy.grid.LazyVerticalGrid import androidx.compose.foundation.lazy.grid.items import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.automirrored.rounded.ArrowBack import androidx.compose.material.icons.rounded.Android import androidx.compose.material.icons.rounded.Bluetooth -import androidx.compose.material.icons.rounded.Search import androidx.compose.material.icons.rounded.Wifi import androidx.compose.material3.BottomAppBar import androidx.compose.material3.CircularProgressIndicator -import androidx.compose.material3.DockedSearchBar import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold -import androidx.compose.material3.SearchBarDefaults import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.saveable.rememberSaveable -import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalLayoutDirection @@ -51,7 +42,8 @@ import io.github.sds100.keymapper.R import io.github.sds100.keymapper.compose.KeyMapperTheme import io.github.sds100.keymapper.util.State import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo -import io.github.sds100.keymapper.util.ui.compose.SimpleListItem +import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions +import io.github.sds100.keymapper.util.ui.compose.SimpleListItemFixedHeight import io.github.sds100.keymapper.util.ui.compose.SimpleListItemGroup import io.github.sds100.keymapper.util.ui.compose.SimpleListItemHeader import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel @@ -92,69 +84,18 @@ private fun ChooseActionScreen( onClickAction: (String) -> Unit = {}, onNavigateBack: () -> Unit = {}, ) { - var isExpanded: Boolean by rememberSaveable { mutableStateOf(false) } - Scaffold( modifier = modifier.displayCutoutPadding(), bottomBar = { BottomAppBar( modifier = Modifier.imePadding(), actions = { - IconButton(onClick = { - if (isExpanded) { - onCloseSearch() - isExpanded = false - } else { - onNavigateBack() - } - }) { - Icon( - Icons.AutoMirrored.Rounded.ArrowBack, - contentDescription = stringResource(R.string.bottom_app_bar_back_content_description), - ) - } - - DockedSearchBar( - modifier = Modifier.align(Alignment.CenterVertically), - inputField = { - SearchBarDefaults.InputField( - modifier = Modifier.align(Alignment.CenterVertically), - onSearch = { - onQueryChange(it) - isExpanded = false - }, - leadingIcon = { - Icon( - Icons.Rounded.Search, - contentDescription = null, - ) - }, - enabled = state is State.Data, - placeholder = { Text(stringResource(R.string.search_placeholder)) }, - query = query ?: "", - onQueryChange = onQueryChange, - expanded = isExpanded, - onExpandedChange = { expanded -> - if (expanded) { - isExpanded = true - } else { - onCloseSearch() - isExpanded = false - } - }, - ) - }, - // This is false to prevent an empty "content" showing underneath. - expanded = isExpanded, - onExpandedChange = { expanded -> - if (expanded) { - isExpanded = true - } else { - onCloseSearch() - isExpanded = false - } - }, - content = {}, + SearchAppBarActions( + onCloseSearch = onCloseSearch, + onNavigateBack = onNavigateBack, + onQueryChange = onQueryChange, + enabled = state is State.Data, + query = query, ) }, ) @@ -260,7 +201,7 @@ private fun ListScreen( group.items, contentType = { "list_item" }, ) { model -> - SimpleListItem( + SimpleListItemFixedHeight( modifier = Modifier.fillMaxWidth(), model = model, onClick = { onClickAction(model.id) }, @@ -363,6 +304,15 @@ private fun PreviewGrid() { isEnabled = false, ), + SimpleListItemModel( + "long", + title = "Very very very very very very very long title", + icon = ComposeIconInfo.Vector(Icons.Rounded.Bluetooth), + subtitle = null, + isSubtitleError = true, + isEnabled = false, + ), + ), ), diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt index f501900c70..96d6da8bb0 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/ConfigActionsViewModel.kt @@ -177,6 +177,7 @@ class ConfigActionsViewModel( override fun onEditClick() { val actionUid = actionOptionsUid.value ?: return coroutineScope.launch { + actionOptionsUid.update { null } val keyMap = config.keyMap.first().dataOrNull() ?: return@launch val oldAction = keyMap.actionList.find { it.uid == actionUid } ?: return@launch diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt index 7d34983a0d..649ef42bd7 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/CreateActionDelegate.kt @@ -786,6 +786,15 @@ class CreateActionDelegate( } return null } + + ActionId.INTERACT_UI_ELEMENT -> { + val oldAction = oldData as? ActionData.InteractUiElement + + return navigate( + "config_interact_ui_element_action", + NavDestination.InteractUiElement(oldAction), + ) + } } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt b/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt index 346dc1114b..1482dd5327 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/FlashlightActionBottomSheet.kt @@ -35,9 +35,7 @@ import androidx.compose.material3.SheetValue import androidx.compose.material3.Slider import androidx.compose.material3.Text import androidx.compose.material3.TextButton -import androidx.compose.material3.TimePicker import androidx.compose.material3.rememberModalBottomSheetState -import androidx.compose.material3.rememberTimePickerState import androidx.compose.runtime.Composable import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt b/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt index 25725b2a41..15d0063245 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/HttpRequestBottomSheet.kt @@ -48,7 +48,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull @OptIn(ExperimentalMaterial3Api::class) @Composable fun HttpRequestBottomSheet(delegate: CreateActionDelegate) { - val scope = rememberCoroutineScope() + rememberCoroutineScope() val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) if (delegate.httpRequestBottomSheetState != null) { @@ -139,10 +139,10 @@ private fun HttpRequestBottomSheet( .padding(horizontal = 16.dp), expanded = methodExpanded, onExpandedChange = { methodExpanded = it }, - value = state.method.toString(), - onValueChanged = { - onSelectMethod(HttpMethod.valueOf(it)) - }, + label = { Text(stringResource(R.string.action_http_request_method_label)) }, + selectedValue = state.method, + values = HttpMethod.entries.map { it to it.toString() }, + onValueChanged = onSelectMethod, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt index 9734435728..80c6ea900d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/actions/PerformActionsUseCase.kt @@ -10,6 +10,7 @@ import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.PreferenceDefaults import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeAction +import io.github.sds100.keymapper.system.accessibility.AccessibilityNodeModel import io.github.sds100.keymapper.system.accessibility.IAccessibilityService import io.github.sds100.keymapper.system.accessibility.ServiceAdapter import io.github.sds100.keymapper.system.airplanemode.AirplaneModeAdapter @@ -65,6 +66,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.stateIn import splitties.bitflags.withFlag @@ -799,6 +801,19 @@ class PerformActionsUseCaseImpl( authorizationHeader = action.authorizationHeader, ) } + + is ActionData.InteractUiElement -> { + if (accessibilityService.activeWindowPackage.first() != action.packageName) { + result = Error.UiElementNotFound + } else { + result = accessibilityService.performActionOnNode( + findNode = { node -> + matchAccessibilityNode(node, action) + }, + performAction = { AccessibilityNodeAction(action = action.nodeAction.accessibilityActionId) }, + ) + } + } } when (result) { @@ -891,6 +906,45 @@ class PerformActionsUseCaseImpl( popupMessageAdapter.showPopupMessage(it.getFullMessage(resourceProvider)) } } + + private fun matchAccessibilityNode( + node: AccessibilityNodeModel, + action: ActionData.InteractUiElement, + ): Boolean { + if (compareIfNonNull(node.uniqueId, action.uniqueId)) { + return true + } + + val viewResourceIdMatches = node.viewResourceId == action.viewResourceId + val classNameMatches = node.className == action.className + + if (compareIfNonNull( + node.contentDescription, + action.contentDescription, + ) && + viewResourceIdMatches && + classNameMatches + ) { + return true + } + + if (compareIfNonNull(node.text, action.text) && + viewResourceIdMatches && + classNameMatches + ) { + return true + } + + if (viewResourceIdMatches) { + return true + } + + return false + } + + private fun compareIfNonNull(a: T?, b: T?): Boolean { + return a != null && b != null && a == b + } } interface PerformActionsUseCase { diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt new file mode 100644 index 0000000000..612cb913af --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/ChooseUiElementScreen.kt @@ -0,0 +1,365 @@ +package io.github.sds100.keymapper.actions.uielement + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.calculateEndPadding +import androidx.compose.foundation.layout.calculateStartPadding +import androidx.compose.foundation.layout.displayCutoutPadding +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.rounded.ErrorOutline +import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedCard +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.ui.compose.KeyMapperDropdownMenu +import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions + +@Composable +fun ChooseElementScreen( + modifier: Modifier = Modifier, + state: State, + query: String?, + onCloseSearch: () -> Unit = {}, + onNavigateBack: () -> Unit = {}, + onQueryChange: (String) -> Unit = {}, + onClickElement: (Long) -> Unit = {}, + onSelectInteractionType: (NodeInteractionType?) -> Unit = {}, +) { + var interactionTypeExpanded by rememberSaveable { mutableStateOf(false) } + + Scaffold( + modifier.displayCutoutPadding(), + bottomBar = { + BottomAppBar( + modifier = Modifier.imePadding(), + actions = { + SearchAppBarActions( + onCloseSearch = onCloseSearch, + onNavigateBack = onNavigateBack, + onQueryChange = onQueryChange, + enabled = state is State.Data, + query = query, + ) + }, + ) + }, + ) { innerPadding -> + val layoutDirection = LocalLayoutDirection.current + val startPadding = innerPadding.calculateStartPadding(layoutDirection) + val endPadding = innerPadding.calculateEndPadding(layoutDirection) + + Surface( + modifier = Modifier + .fillMaxSize() + .padding( + top = innerPadding.calculateTopPadding(), + bottom = innerPadding.calculateBottomPadding(), + start = startPadding, + end = endPadding, + ), + ) { + Column { + Text( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = 16.dp, + bottom = 8.dp, + ), + text = stringResource(R.string.action_interact_ui_element_choose_element_title), + style = MaterialTheme.typography.titleLarge, + ) + + Text( + modifier = Modifier.padding(horizontal = 16.dp), + text = stringResource(R.string.action_interact_ui_element_choose_element_text), + style = MaterialTheme.typography.bodyMedium, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Row( + modifier = Modifier.padding(horizontal = 16.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon( + imageVector = Icons.Rounded.ErrorOutline, + contentDescription = null, + tint = MaterialTheme.colorScheme.error, + ) + Spacer(modifier = Modifier.width(8.dp)) + Text( + text = stringResource(R.string.action_interact_ui_element_choose_element_not_found_subtitle), + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.titleSmall, + ) + } + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + modifier = Modifier.padding(horizontal = 16.dp), + text = stringResource(R.string.action_interact_ui_element_choose_element_not_found_text), + style = MaterialTheme.typography.bodyMedium, + ) + Spacer(modifier = Modifier.height(8.dp)) + + when (state) { + State.Loading -> LoadingList(modifier = Modifier.fillMaxSize()) + is State.Data -> { + val listItems = state.data.listItems + + if (listItems.isEmpty()) { + EmptyList(modifier = Modifier.fillMaxSize()) + } else { + KeyMapperDropdownMenu( + modifier = Modifier.padding(horizontal = 16.dp), + expanded = interactionTypeExpanded, + onExpandedChange = { interactionTypeExpanded = it }, + label = { Text(stringResource(R.string.action_interact_ui_element_filter_interaction_type_dropdown)) }, + values = state.data.interactionTypes, + selectedValue = state.data.selectedInteractionType, + onValueChanged = onSelectInteractionType, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + LoadedList( + modifier = Modifier.fillMaxSize(), + listItems = listItems, + onClick = onClickElement, + ) + } + } + } + } + } + } +} + +@Composable +private fun LoadingList(modifier: Modifier = Modifier) { + Box(modifier) { + CircularProgressIndicator(modifier = Modifier.align(Alignment.Center)) + } +} + +@Composable +private fun EmptyList(modifier: Modifier = Modifier) { + Box(modifier) { + val shrug = stringResource(R.string.shrug) + val text = stringResource(R.string.ui_element_list_empty) + Text( + modifier = Modifier.align(Alignment.Center), + text = buildAnnotatedString { + withStyle(MaterialTheme.typography.headlineLarge.toSpanStyle()) { + append(shrug) + } + appendLine() + appendLine() + withStyle(MaterialTheme.typography.bodyLarge.toSpanStyle()) { + append(text) + } + }, + textAlign = TextAlign.Center, + ) + } +} + +@Composable +private fun LoadedList( + modifier: Modifier = Modifier, + listItems: List, + onClick: (Long) -> Unit, +) { + LazyColumn( + modifier = modifier, + contentPadding = PaddingValues(8.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + items(listItems, key = { it.id }) { model -> + UiElementListItem( + modifier = Modifier.fillMaxWidth(), + model = model, + onClick = { onClick(model.id) }, + ) + } + } +} + +@Composable +private fun UiElementListItem( + modifier: Modifier = Modifier, + model: UiElementListItemModel, + onClick: () -> Unit, +) { + OutlinedCard(modifier = modifier, onClick = onClick) { + Column( + modifier = Modifier.padding(16.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + if (model.nodeViewResourceId != null) { + Text( + text = "View ID: ${model.nodeViewResourceId}", + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.primary, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + + if (model.nodeText != null) { + Text( + text = "\"${model.nodeText}\"", + style = MaterialTheme.typography.titleMedium.copy(fontWeight = FontWeight.Bold), + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + + if (model.nodeClassName != null) { + TextWithLeadingLabel( + title = stringResource(R.string.action_interact_ui_element_class_name_label), + text = model.nodeClassName, + ) + } + + if (model.nodeUniqueId != null) { + TextWithLeadingLabel( + title = stringResource(R.string.action_interact_ui_element_unique_id_label), + text = model.nodeUniqueId, + ) + } + + TextWithLeadingLabel( + title = stringResource(R.string.action_interact_ui_element_interaction_types_label), + text = model.interactionTypesText, + ) + } + } +} + +@Composable +private fun TextWithLeadingLabel( + modifier: Modifier = Modifier, + title: String, + text: String, +) { + val text = buildAnnotatedString { + pushStyle( + MaterialTheme.typography.bodySmall.copy(fontWeight = FontWeight.Bold).toSpanStyle(), + ) + append(title) + pop() + append(": ") + append(text) + } + + Text( + modifier = modifier, + text = text, + style = MaterialTheme.typography.bodySmall, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) +} + +@Preview +@Composable +private fun Empty() { + KeyMapperTheme { + ChooseElementScreen( + state = State.Data( + SelectUiElementState( + listItems = emptyList(), + interactionTypes = emptyList(), + selectedInteractionType = null, + ), + ), + query = "Key Mapper", + ) + } +} + +@Preview +@Composable +private fun Loading() { + KeyMapperTheme { + ChooseElementScreen( + state = State.Loading, + query = null, + ) + } +} + +@Preview +@Composable +private fun Loaded() { + val listItems = listOf( + UiElementListItemModel( + id = 1L, + nodeText = "Open Settings", + nodeClassName = "android.widget.ImageButton", + nodeViewResourceId = "menu_button", + nodeUniqueId = "123456789", + interactionTypesText = "Tap, Tap and hold, Scroll forward", + interactionTypes = setOf( + NodeInteractionType.CLICK, + NodeInteractionType.LONG_CLICK, + NodeInteractionType.SCROLL_FORWARD, + ), + ), + ) + + val state = SelectUiElementState( + listItems = listItems, + interactionTypes = listOf( + null to "Any", + NodeInteractionType.CLICK to "Tap", + NodeInteractionType.LONG_CLICK to "Tap and hold", + ), + selectedInteractionType = null, + ) + + KeyMapperTheme { + ChooseElementScreen( + state = State.Data(state), + query = "Key Mapper", + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt new file mode 100644 index 0000000000..ddde228691 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementFragment.kt @@ -0,0 +1,90 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.os.Bundle +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.ViewCompositionStrategy +import androidx.core.os.bundleOf +import androidx.fragment.app.Fragment +import androidx.fragment.app.setFragmentResult +import androidx.fragment.app.viewModels +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.withStateAtLeast +import androidx.navigation.findNavController +import androidx.navigation.fragment.findNavController +import androidx.navigation.fragment.navArgs +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.databinding.FragmentComposeBinding +import io.github.sds100.keymapper.util.Inject +import io.github.sds100.keymapper.util.launchRepeatOnLifecycle +import io.github.sds100.keymapper.util.ui.showPopups +import io.github.sds100.keymapper.util.viewLifecycleScope +import kotlinx.coroutines.flow.collectLatest +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json + +class InteractUiElementFragment : Fragment() { + + companion object { + const val EXTRA_ACTION = "extra_action" + } + + private val args: InteractUiElementFragmentArgs by navArgs() + + private val viewModel by viewModels { + Inject.interactUiElementViewModel(requireContext()) + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + args.action?.let { argsAction -> viewModel.loadAction(Json.decodeFromString(argsAction)) } + + launchRepeatOnLifecycle(Lifecycle.State.CREATED) { + viewModel.returnAction.collectLatest { action -> + viewLifecycleScope.launch { + withStateAtLeast(Lifecycle.State.RESUMED) { + setFragmentResult( + args.requestKey, + bundleOf(EXTRA_ACTION to Json.encodeToString(action)), + ) + findNavController().navigateUp() + } + } + } + } + } + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle?, + ): View { + FragmentComposeBinding.inflate(inflater, container, false).apply { + composeView.apply { + // Dispose of the Composition when the view's LifecycleOwner + // is destroyed + setViewCompositionStrategy(ViewCompositionStrategy.DisposeOnViewTreeLifecycleDestroyed) + setContent { + KeyMapperTheme { + InteractUiElementScreen( + modifier = Modifier.fillMaxSize(), + viewModel = viewModel, + navigateBack = findNavController()::navigateUp, + ) + } + } + } + return this.root + } + } + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + super.onViewCreated(view, savedInstanceState) + + viewModel.showPopups(this, view) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt new file mode 100644 index 0000000000..c4fd683a5d --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementScreen.kt @@ -0,0 +1,631 @@ +package io.github.sds100.keymapper.actions.uielement + +import androidx.activity.compose.BackHandler +import androidx.compose.animation.AnimatedContentTransitionScope +import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.calculateEndPadding +import androidx.compose.foundation.layout.calculateStartPadding +import androidx.compose.foundation.layout.displayCutoutPadding +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.sizeIn +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.rounded.ArrowBack +import androidx.compose.material.icons.outlined.Info +import androidx.compose.material.icons.rounded.Check +import androidx.compose.material.icons.rounded.ChevronRight +import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.ExtendedFloatingActionButton +import androidx.compose.material3.FilledTonalButton +import androidx.compose.material3.FloatingActionButtonDefaults +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.LocalContentColor +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Scaffold +import androidx.compose.material3.SnackbarHost +import androidx.compose.material3.SnackbarHostState +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.contentColorFor +import androidx.compose.runtime.Composable +import androidx.compose.runtime.CompositionLocalProvider +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.pluralStringResource +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.navigation.compose.NavHost +import androidx.navigation.compose.composable +import androidx.navigation.compose.rememberNavController +import com.google.accompanist.drawablepainter.rememberDrawablePainter +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.compose.LocalCustomColorsPalette +import io.github.sds100.keymapper.system.apps.ChooseAppScreen +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.drawable +import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.KeyMapperDropdownMenu +import io.github.sds100.keymapper.util.ui.compose.OptionsHeaderRow +import io.github.sds100.keymapper.util.ui.compose.icons.AdGroup +import io.github.sds100.keymapper.util.ui.compose.icons.JumpToElement +import io.github.sds100.keymapper.util.ui.compose.icons.KeyMapperIcons +import kotlinx.coroutines.flow.update + +private const val DEST_LANDING = "landing" +private const val DEST_SELECT_APP = "select_app" +private const val DEST_SELECT_ELEMENT = "select_element" + +@Composable +fun InteractUiElementScreen( + modifier: Modifier = Modifier, + viewModel: InteractUiElementViewModel, + navigateBack: () -> Unit, +) { + val navController = rememberNavController() + + val recordState by viewModel.recordState.collectAsStateWithLifecycle() + val selectedElementState by viewModel.selectedElementState.collectAsStateWithLifecycle() + + val appListState by viewModel.filteredAppListItems.collectAsStateWithLifecycle() + val appSearchQuery by viewModel.appSearchQuery.collectAsStateWithLifecycle() + + val elementListState by viewModel.selectUiElementState.collectAsStateWithLifecycle() + val elementSearchQuery by viewModel.elementSearchQuery.collectAsStateWithLifecycle() + + val onBackClick = { + if (!navController.navigateUp()) { + navigateBack() + } + } + + BackHandler(onBack = onBackClick) + + NavHost( + modifier = modifier, + navController = navController, + startDestination = DEST_LANDING, + enterTransition = { slideIntoContainer(towards = AnimatedContentTransitionScope.SlideDirection.Left) }, + exitTransition = { slideOutOfContainer(towards = AnimatedContentTransitionScope.SlideDirection.Right) }, + popEnterTransition = { slideIntoContainer(towards = AnimatedContentTransitionScope.SlideDirection.Right) }, + popExitTransition = { slideOutOfContainer(towards = AnimatedContentTransitionScope.SlideDirection.Right) }, + ) { + composable(DEST_LANDING) { + LandingScreen( + modifier = Modifier.fillMaxSize(), + recordState = recordState, + selectedElementState = selectedElementState, + onRecordClick = viewModel::onRecordClick, + onBackClick = onBackClick, + onDoneClick = viewModel::onDoneClick, + openSelectAppScreen = { + navController.navigate(DEST_SELECT_APP) + }, + onSelectInteractionType = viewModel::onSelectElementInteractionType, + onDescriptionChanged = viewModel::onDescriptionChanged, + ) + } + + composable(DEST_SELECT_APP) { + ChooseAppScreen( + modifier = Modifier.fillMaxSize(), + title = stringResource(R.string.action_interact_ui_element_choose_element_title), + state = appListState, + query = appSearchQuery, + onQueryChange = { query -> viewModel.appSearchQuery.update { query } }, + onCloseSearch = { viewModel.appSearchQuery.update { null } }, + onNavigateBack = onBackClick, + onClickApp = { + viewModel.onSelectApp(it) + navController.navigate(DEST_SELECT_ELEMENT) + }, + ) + } + + composable(DEST_SELECT_ELEMENT) { + ChooseElementScreen( + modifier = Modifier.fillMaxSize(), + state = elementListState, + query = elementSearchQuery, + onCloseSearch = { viewModel.elementSearchQuery.update { null } }, + onNavigateBack = onBackClick, + onQueryChange = { query -> viewModel.elementSearchQuery.update { query } }, + onClickElement = { + viewModel.onSelectElement(it) + navController.popBackStack(route = DEST_LANDING, inclusive = false) + }, + onSelectInteractionType = viewModel::onSelectInteractionTypeFilter, + ) + } + } +} + +@Composable +private fun LandingScreen( + modifier: Modifier = Modifier, + recordState: State, + selectedElementState: SelectedUiElementState?, + onRecordClick: () -> Unit = {}, + onBackClick: () -> Unit = {}, + onDoneClick: () -> Unit = {}, + openSelectAppScreen: () -> Unit = {}, + onSelectInteractionType: (NodeInteractionType) -> Unit = {}, + onDescriptionChanged: (String) -> Unit = {}, +) { + val snackbarHostState = SnackbarHostState() + + Scaffold( + modifier.displayCutoutPadding(), + snackbarHost = { SnackbarHost(snackbarHostState) }, + bottomBar = { + BottomAppBar(actions = { + IconButton(onClick = onBackClick) { + Icon( + Icons.AutoMirrored.Rounded.ArrowBack, + stringResource(R.string.action_go_back), + ) + } + }, floatingActionButton = { + if (selectedElementState == null || selectedElementState.description.isBlank()) { + DisabledExtendedFloatingActionButton( + icon = { Icon(Icons.Rounded.Check, stringResource(R.string.button_done)) }, + text = stringResource(R.string.button_done), + ) + } else { + ExtendedFloatingActionButton( + onClick = onDoneClick, + text = { Text(stringResource(R.string.button_done)) }, + icon = { + Icon(Icons.Rounded.Check, stringResource(R.string.button_done)) + }, + elevation = FloatingActionButtonDefaults.bottomAppBarFabElevation(), + ) + } + }) + }, + ) { innerPadding -> + val layoutDirection = LocalLayoutDirection.current + val startPadding = innerPadding.calculateStartPadding(layoutDirection) + val endPadding = innerPadding.calculateEndPadding(layoutDirection) + + Surface( + modifier = Modifier + .fillMaxSize() + .padding( + top = innerPadding.calculateTopPadding(), + bottom = innerPadding.calculateBottomPadding(), + start = startPadding, + end = endPadding, + ), + ) { + Column(modifier = Modifier.verticalScroll(rememberScrollState())) { + Text( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = 16.dp, + bottom = 8.dp, + ), + text = stringResource(R.string.action_interact_ui_element_title), + style = MaterialTheme.typography.titleLarge, + ) + + Text( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + text = stringResource(R.string.action_interact_ui_element_description), + style = MaterialTheme.typography.bodyMedium, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + RecordingSection( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + state = recordState, + onRecordClick = onRecordClick, + openSelectAppScreen = openSelectAppScreen, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + if (selectedElementState != null) { + HorizontalDivider( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + ) + + Spacer(modifier = Modifier.height(8.dp)) + + SelectedElementSection( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp), + state = selectedElementState, + onSelectInteractionType = onSelectInteractionType, + onDescriptionChanged = onDescriptionChanged, + ) + } + } + } + } +} + +@Composable +private fun DisabledExtendedFloatingActionButton( + modifier: Modifier = Modifier, + icon: @Composable () -> Unit, + text: String, +) { + Surface( + modifier = modifier, + shape = FloatingActionButtonDefaults.extendedFabShape, + color = FloatingActionButtonDefaults.containerColor.copy(alpha = 0.5f), + ) { + Row( + modifier = + Modifier + .sizeIn(minWidth = 80.dp, minHeight = 56.dp) + .padding(start = 16.dp, end = 20.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.Start, + ) { + val contentColor = + MaterialTheme.colorScheme.contentColorFor(FloatingActionButtonDefaults.containerColor) + .copy(alpha = 0.5f) + + CompositionLocalProvider(LocalContentColor provides contentColor) { + icon() + Spacer(Modifier.width(12.dp)) + Text( + text, + style = MaterialTheme.typography.labelLarge, + ) + } + } + } +} + +@Composable +private fun RecordingSection( + modifier: Modifier = Modifier, + state: State, + onRecordClick: () -> Unit = {}, + openSelectAppScreen: () -> Unit = {}, +) { + Column(modifier = modifier) { + when (state) { + is State.Data -> { + val interactionCount: Int = when (state.data) { + is RecordUiElementState.CountingDown -> state.data.interactionCount + is RecordUiElementState.Recorded -> state.data.interactionCount + RecordUiElementState.Empty -> 0 + } + + InteractionCountBox( + modifier = Modifier.fillMaxWidth(), + interactionCount = interactionCount, + onClick = openSelectAppScreen, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + RecordButton( + modifier = Modifier.fillMaxWidth(), + state = state.data, + onClick = onRecordClick, + ) + } + + State.Loading -> { + Spacer(modifier = Modifier.height(16.dp)) + LinearProgressIndicator(modifier = Modifier.fillMaxWidth()) + Spacer(modifier = Modifier.height(16.dp)) + } + } + } +} + +@Composable +private fun InteractionCountBox( + modifier: Modifier = Modifier, + interactionCount: Int, + onClick: () -> Unit, +) { + val enabled = interactionCount > 0 + + val color = if (enabled) { + MaterialTheme.colorScheme.onSurface + } else { + MaterialTheme.colorScheme.onSurface.copy(alpha = 0.5f) + } + + Surface( + modifier = modifier, + onClick = onClick, + enabled = enabled, + shape = MaterialTheme.shapes.medium, + ) { + CompositionLocalProvider( + LocalContentColor provides color, + ) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon(imageVector = KeyMapperIcons.AdGroup, contentDescription = null) + Spacer(modifier = Modifier.width(16.dp)) + Column(modifier = Modifier.weight(1f)) { + Text( + pluralStringResource( + R.plurals.action_interact_ui_element_interactions_detected, + interactionCount, + interactionCount, + ), + style = MaterialTheme.typography.bodyLarge, + ) + + Text( + stringResource(R.string.action_interact_ui_element_choose_interaction), + style = MaterialTheme.typography.bodyMedium, + ) + } + + Spacer(modifier = Modifier.width(16.dp)) + Icon(imageVector = Icons.Rounded.ChevronRight, contentDescription = null) + } + } + } +} + +@Composable +private fun RecordButton( + modifier: Modifier, + state: RecordUiElementState, + onClick: () -> Unit, +) { + val text: String = when (state) { + is RecordUiElementState.Empty -> stringResource(R.string.action_interact_ui_element_start_recording) + is RecordUiElementState.Recorded -> stringResource(R.string.action_interact_ui_element_record_again) + is RecordUiElementState.CountingDown -> stringResource( + R.string.action_interact_ui_element_stop_recording, + state.timeRemaining, + ) + } + + if (state is RecordUiElementState.Recorded) { + OutlinedButton( + modifier = modifier, + onClick = onClick, + colors = ButtonDefaults.outlinedButtonColors().copy( + contentColor = LocalCustomColorsPalette.current.red, + ), + border = BorderStroke(1.dp, color = LocalCustomColorsPalette.current.red), + ) { + Text( + text = text, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } else { + FilledTonalButton( + modifier = modifier, + onClick = onClick, + colors = ButtonDefaults.filledTonalButtonColors().copy( + containerColor = LocalCustomColorsPalette.current.red, + contentColor = LocalCustomColorsPalette.current.onRed, + ), + ) { + Text( + text = text, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } +} + +@Composable +private fun SelectedElementSection( + modifier: Modifier = Modifier, + state: SelectedUiElementState, + onDescriptionChanged: (String) -> Unit = {}, + onSelectInteractionType: (NodeInteractionType) -> Unit = {}, +) { + var interactionTypeExpanded by rememberSaveable { mutableStateOf(false) } + + Column(modifier = modifier) { + val isError = state.description.isBlank() + + OutlinedTextField( + modifier = Modifier.fillMaxWidth(), + value = state.description, + onValueChange = onDescriptionChanged, + isError = isError, + supportingText = if (isError) { + { Text(stringResource(R.string.error_cant_be_empty)) } + } else { + null + }, + label = { + Text(stringResource(R.string.action_interact_ui_element_description_label)) + }, + ) + + Spacer(modifier = Modifier.height(16.dp)) + + OptionsHeaderRow( + icon = Icons.Outlined.Info, + text = stringResource(R.string.action_interact_ui_element_interaction_details_title), + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = stringResource(R.string.action_interact_ui_element_app_label), + style = MaterialTheme.typography.titleSmall, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) { + if (state.appIcon != null) { + val painter = rememberDrawablePainter(state.appIcon.drawable) + Icon( + modifier = Modifier.size(24.dp), + painter = painter, + contentDescription = null, + tint = Color.Unspecified, + ) + Spacer(modifier = Modifier.width(8.dp)) + Text(text = state.appName, style = MaterialTheme.typography.bodyMedium) + } + } + + Spacer(modifier = Modifier.height(8.dp)) + + if (state.nodeText != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_text_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeText, style = MaterialTheme.typography.bodyMedium) + } + + Spacer(modifier = Modifier.height(8.dp)) + + if (state.nodeClassName != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_class_name_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeClassName, style = MaterialTheme.typography.bodyMedium) + Spacer(modifier = Modifier.height(8.dp)) + } + + if (state.nodeViewResourceId != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_view_id_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeViewResourceId, style = MaterialTheme.typography.bodyMedium) + + Spacer(modifier = Modifier.height(8.dp)) + } + + if (state.nodeUniqueId != null) { + Text( + text = stringResource(R.string.action_interact_ui_element_unique_id_label), + style = MaterialTheme.typography.titleSmall, + ) + + Text(text = state.nodeUniqueId, style = MaterialTheme.typography.bodyMedium) + Spacer(modifier = Modifier.height(8.dp)) + } + + OptionsHeaderRow( + icon = KeyMapperIcons.JumpToElement, + text = stringResource(R.string.action_interact_ui_element_interaction_type_dropdown), + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = stringResource(R.string.action_interact_ui_element_interaction_type_dropdown_caption), + style = MaterialTheme.typography.bodyMedium, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + KeyMapperDropdownMenu( + expanded = interactionTypeExpanded, + onExpandedChange = { interactionTypeExpanded = it }, + values = state.interactionTypes, + selectedValue = state.selectedInteraction, + onValueChanged = onSelectInteractionType, + ) + + Spacer(modifier = Modifier.height(8.dp)) + } +} + +@Preview +@Composable +private fun PreviewEmpty() { + KeyMapperTheme { + LandingScreen( + recordState = State.Data(RecordUiElementState.Empty), + selectedElementState = null, + ) + } +} + +@Preview +@Composable +private fun PreviewSelectedElement() { + val appIcon = LocalContext.current.drawable(R.mipmap.ic_launcher_round) + + KeyMapperTheme { + LandingScreen( + recordState = State.Data(RecordUiElementState.Recorded(3)), + selectedElementState = SelectedUiElementState( + description = "Tap test node", + packageName = "com.example.test", + appName = "Test App", + appIcon = ComposeIconInfo.Drawable(appIcon), + nodeText = "Test Node", + nodeClassName = "android.widget.ImageButton", + nodeViewResourceId = "io.github.sds100.keymapper:id/menu_button", + nodeUniqueId = "123", + interactionTypes = listOf(NodeInteractionType.LONG_CLICK to "Tap and hold"), + selectedInteraction = NodeInteractionType.LONG_CLICK, + ), + ) + } +} + +@Preview +@Composable +private fun PreviewLoading() { + KeyMapperTheme { + LandingScreen( + recordState = State.Loading, + selectedElementState = null, + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt new file mode 100644 index 0000000000..9a325f22f3 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementUseCase.kt @@ -0,0 +1,96 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.graphics.drawable.Drawable +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository +import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState +import io.github.sds100.keymapper.system.accessibility.ServiceAdapter +import io.github.sds100.keymapper.system.apps.PackageManagerAdapter +import io.github.sds100.keymapper.util.Result +import io.github.sds100.keymapper.util.ServiceEvent +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.mapData +import io.github.sds100.keymapper.util.onFailure +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.update + +class InteractUiElementController( + private val coroutineScope: CoroutineScope, + private val serviceAdapter: ServiceAdapter, + private val nodeRepository: AccessibilityNodeRepository, + private val packageManagerAdapter: PackageManagerAdapter, +) : InteractUiElementUseCase { + override val recordState: MutableStateFlow = + MutableStateFlow(RecordAccessibilityNodeState.Idle) + + override val interactionCount: Flow> = + nodeRepository.nodes.map { state -> state.mapData { it.size } } + + override val interactedPackages: Flow>> = nodeRepository.nodes.map { state -> + state.mapData { nodes -> + nodes.map { it.packageName }.distinct() + } + } + + init { + serviceAdapter.eventReceiver + .filterIsInstance() + .onEach { event -> recordState.update { event.state } } + .launchIn(coroutineScope) + } + + override fun getInteractionsByPackage(packageName: String): Flow>> { + return nodeRepository.nodes.map { state -> + state.mapData { nodes -> + nodes.filter { it.packageName == packageName } + } + } + } + + override suspend fun getInteractionById(id: Long): AccessibilityNodeEntity? { + return nodeRepository.get(id) + } + + override fun getAppName(packageName: String): Result = packageManagerAdapter.getAppName(packageName) + + override fun getAppIcon(packageName: String): Result = packageManagerAdapter.getAppIcon(packageName) + + override suspend fun startRecording(): Result<*> { + nodeRepository.deleteAll() + return serviceAdapter.send(ServiceEvent.StartRecordingNodes) + } + + override suspend fun stopRecording() { + serviceAdapter.send(ServiceEvent.StopRecordingNodes).onFailure { + recordState.update { RecordAccessibilityNodeState.Idle } + } + } + + override fun startService(): Boolean { + return serviceAdapter.start() + } +} + +interface InteractUiElementUseCase { + val recordState: StateFlow + + val interactionCount: Flow> + val interactedPackages: Flow>> + fun getInteractionsByPackage(packageName: String): Flow>> + suspend fun getInteractionById(id: Long): AccessibilityNodeEntity? + + fun getAppName(packageName: String): Result + fun getAppIcon(packageName: String): Result + + suspend fun startRecording(): Result<*> + suspend fun stopRecording() + + fun startService(): Boolean +} diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt new file mode 100644 index 0000000000..80d255273b --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/InteractUiElementViewModel.kt @@ -0,0 +1,412 @@ +package io.github.sds100.keymapper.actions.uielement + +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.rounded.Android +import androidx.lifecycle.ViewModel +import androidx.lifecycle.ViewModelProvider +import androidx.lifecycle.viewModelScope +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.actions.ActionData +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState +import io.github.sds100.keymapper.util.Error +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.Success +import io.github.sds100.keymapper.util.containsQuery +import io.github.sds100.keymapper.util.dataOrNull +import io.github.sds100.keymapper.util.ifIsData +import io.github.sds100.keymapper.util.mapData +import io.github.sds100.keymapper.util.onFailure +import io.github.sds100.keymapper.util.then +import io.github.sds100.keymapper.util.ui.PopupViewModel +import io.github.sds100.keymapper.util.ui.PopupViewModelImpl +import io.github.sds100.keymapper.util.ui.ResourceProvider +import io.github.sds100.keymapper.util.ui.ViewModelHelper +import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel +import io.github.sds100.keymapper.util.valueOrNull +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.flatMapLatest +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch +import java.util.Locale + +class InteractUiElementViewModel( + private val useCase: InteractUiElementUseCase, + private val resourceProvider: ResourceProvider, +) : ViewModel(), + ResourceProvider by resourceProvider, + PopupViewModel by PopupViewModelImpl() { + + private val _returnAction: MutableSharedFlow = MutableSharedFlow() + val returnAction: SharedFlow = _returnAction.asSharedFlow() + + val recordState: StateFlow> = combine( + useCase.recordState, + useCase.interactionCount, + ) { recordState, interactionCountState -> + val interactionCount = interactionCountState.dataOrNull() ?: return@combine State.Loading + + when (recordState) { + is RecordAccessibilityNodeState.CountingDown -> { + val mins = recordState.timeLeft / 60 + val secs = recordState.timeLeft % 60 + + val timeRemainingText = String.format( + Locale.getDefault(), + "%02d:%02d", + mins, + secs, + ) + + State.Data( + RecordUiElementState.CountingDown( + timeRemaining = timeRemainingText, + interactionCount = interactionCount, + ), + ) + } + + RecordAccessibilityNodeState.Idle -> { + if (interactionCount == 0) { + State.Data(RecordUiElementState.Empty) + } else { + State.Data(RecordUiElementState.Recorded(interactionCount = interactionCount)) + } + } + } + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + private val _selectedElementState = MutableStateFlow(null) + val selectedElementState: StateFlow = + _selectedElementState.asStateFlow() + + val appSearchQuery = MutableStateFlow(null) + + private val appListItems: Flow>> = useCase.interactedPackages + .map { state -> state.mapData { list -> list.map(::buildInteractedPackageListItem) } } + + val filteredAppListItems = combine( + appListItems, + appSearchQuery, + ) { state, query -> + state.mapData { listItems -> + listItems.filter { model -> + model.title.containsQuery(query) + } + } + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + private val selectedApp = MutableStateFlow(null) + + val elementSearchQuery = MutableStateFlow(null) + + @OptIn(ExperimentalCoroutinesApi::class) + private val interactionsByPackage: StateFlow>> = selectedApp + .filterNotNull() + .flatMapLatest { packageName -> useCase.getInteractionsByPackage(packageName) } + .stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + private val elementListItems: Flow>> = interactionsByPackage + .map { state -> state.mapData { list -> list.map(::buildUiElementListItem) } } + + private val interactionTypesFilterItems: Flow>>> = + interactionsByPackage + .map { state -> + state.mapData { list -> + val any = Pair( + null, + getString(R.string.action_interact_ui_element_interaction_type_any), + ) + + val interactionTypes = list.flatMap { it.actions }.toSet() + + listOf(any).plus(buildInteractionTypeFilterItems(interactionTypes)) + } + } + + private val selectedInteractionTypeFilter = MutableStateFlow(null) + + private val filteredElementListItems = combine( + elementListItems, + elementSearchQuery, + selectedInteractionTypeFilter, + ) { state, query, interactionType -> + state.mapData { listItems -> + listItems.filter { model -> + if (interactionType != null && !model.interactionTypes.contains(interactionType)) { + return@filter false + } + + val modelString = buildString { + append(model.nodeText) + append(" ") + append(model.nodeClassName) + append(" ") + append(model.nodeViewResourceId) + } + modelString.containsQuery(query) + } + } + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + val selectUiElementState: StateFlow> = combine( + filteredElementListItems, + interactionTypesFilterItems, + selectedInteractionTypeFilter, + ) { listItemsState, interactionTypesState, selectedInteractionType -> + val listItems = listItemsState.dataOrNull() ?: return@combine State.Loading + val interactionTypes = interactionTypesState.dataOrNull() ?: return@combine State.Loading + + val newState = SelectUiElementState( + listItems = listItems, + interactionTypes = interactionTypes, + selectedInteractionType = selectedInteractionType, + ) + State.Data(newState) + }.stateIn(viewModelScope, SharingStarted.Lazily, State.Loading) + + fun loadAction(action: ActionData.InteractUiElement) { + viewModelScope.launch { + val appName = useCase.getAppName(action.packageName).valueOrNull() ?: action.packageName + val appIcon = getAppIcon(action.packageName) + + val newState = SelectedUiElementState( + description = action.description, + packageName = action.packageName, + appName = appName, + appIcon = appIcon, + nodeText = action.text ?: action.contentDescription, + nodeClassName = action.className, + nodeViewResourceId = action.viewResourceId, + nodeUniqueId = action.uniqueId, + interactionTypes = buildInteractionTypeFilterItems(action.nodeActions), + selectedInteraction = action.nodeAction, + ) + + _selectedElementState.update { newState } + } + } + + fun onDoneClick() { + val selectedElementState = _selectedElementState.value + if (selectedElementState == null) { + return + } + + if (selectedElementState.description.isBlank()) { + return + } + + val action = ActionData.InteractUiElement( + description = selectedElementState.description, + nodeAction = selectedElementState.selectedInteraction, + packageName = selectedElementState.packageName, + text = selectedElementState.nodeText, + contentDescription = selectedElementState.nodeText, + className = selectedElementState.nodeClassName, + viewResourceId = selectedElementState.nodeViewResourceId, + uniqueId = selectedElementState.nodeUniqueId, + nodeActions = selectedElementState.interactionTypes.map { it.first }.toSet(), + ) + + viewModelScope.launch { + _returnAction.emit(action) + } + } + + fun onRecordClick() { + recordState.value.ifIsData { recordState -> + viewModelScope.launch { + when (recordState) { + is RecordUiElementState.CountingDown -> useCase.stopRecording() + RecordUiElementState.Empty -> startRecording() + is RecordUiElementState.Recorded -> startRecording() + } + } + } + } + + fun onSelectApp(packageName: String) { + elementSearchQuery.update { null } + selectedApp.update { packageName } + } + + fun onSelectElement(id: Long) { + viewModelScope.launch { + val interaction = useCase.getInteractionById(id) ?: return@launch + + val appName = + useCase.getAppName(interaction.packageName).valueOrNull() ?: interaction.packageName + val appIcon = getAppIcon(interaction.packageName) + + val selectedInteraction = + NodeInteractionType.entries.first { interaction.actions.contains(it) } + + val newState = SelectedUiElementState( + description = "", + packageName = interaction.packageName, + appName = appName, + appIcon = appIcon, + nodeText = interaction.text ?: interaction.contentDescription, + nodeClassName = interaction.className, + nodeViewResourceId = interaction.viewResourceId, + nodeUniqueId = interaction.uniqueId, + interactionTypes = buildInteractionTypeFilterItems(interaction.actions), + selectedInteraction = selectedInteraction, + ) + + _selectedElementState.update { newState } + } + } + + fun onSelectElementInteractionType(interactionType: NodeInteractionType) { + _selectedElementState.update { state -> + state?.copy(selectedInteraction = interactionType) + } + } + + fun onSelectInteractionTypeFilter(interactionType: NodeInteractionType?) { + selectedInteractionTypeFilter.update { interactionType } + } + + fun onDescriptionChanged(description: String) { + _selectedElementState.update { state -> + state?.copy(description = description) + } + } + + private suspend fun startRecording() { + useCase.startRecording().onFailure { error -> + if (error == Error.AccessibilityServiceDisabled) { + ViewModelHelper.handleAccessibilityServiceStoppedDialog( + this, + this, + startService = { useCase.startService() }, + ) + } else if (error == Error.AccessibilityServiceCrashed) { + ViewModelHelper.handleAccessibilityServiceCrashedDialog( + this, + this, + restartService = { useCase.startService() }, + ) + } + } + } + + private fun buildInteractedPackageListItem(packageName: String): SimpleListItemModel { + val appName = useCase.getAppName(packageName).valueOrNull() ?: packageName + val appIcon = getAppIcon(packageName) ?: ComposeIconInfo.Vector(Icons.Rounded.Android) + + return SimpleListItemModel( + id = packageName, + title = appName, + icon = appIcon, + ) + } + + private fun buildUiElementListItem(node: AccessibilityNodeEntity): UiElementListItemModel { + val resourceIdText = node.viewResourceId?.split("/")?.lastOrNull() + + return UiElementListItemModel( + id = node.id, + nodeViewResourceId = resourceIdText, + nodeText = node.text ?: node.contentDescription, + nodeClassName = node.className, + nodeUniqueId = node.uniqueId?.toString(), + interactionTypesText = node.actions.joinToString { getInteractionTypeString(it) }, + interactionTypes = node.actions, + ) + } + + private fun buildInteractionTypeFilterItems(interactionTypes: Set): List> { + return buildList { + // They should always be in the same order so iterate over the Enum entries. + for (type in NodeInteractionType.entries) { + if (interactionTypes.contains(type)) { + add(type to getInteractionTypeString(type)) + } + } + } + } + + private fun getAppIcon(packageName: String): ComposeIconInfo.Drawable? = useCase + .getAppIcon(packageName) + .then { Success(ComposeIconInfo.Drawable(it)) } + .valueOrNull() + + private fun getInteractionTypeString(interactionType: NodeInteractionType): String { + return when (interactionType) { + NodeInteractionType.CLICK -> getString(R.string.action_interact_ui_element_interaction_type_click) + NodeInteractionType.LONG_CLICK -> getString(R.string.action_interact_ui_element_interaction_type_long_click) + NodeInteractionType.FOCUS -> getString(R.string.action_interact_ui_element_interaction_type_focus) + NodeInteractionType.SELECT -> getString(R.string.action_interact_ui_element_interaction_type_select) + NodeInteractionType.SCROLL_FORWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_forward) + NodeInteractionType.SCROLL_BACKWARD -> getString(R.string.action_interact_ui_element_interaction_type_scroll_backward) + NodeInteractionType.EXPAND -> getString(R.string.action_interact_ui_element_interaction_type_expand) + NodeInteractionType.COLLAPSE -> getString(R.string.action_interact_ui_element_interaction_type_collapse) + } + } + + @Suppress("UNCHECKED_CAST") + class Factory( + private val useCase: InteractUiElementUseCase, + private val resourceProvider: ResourceProvider, + ) : ViewModelProvider.NewInstanceFactory() { + override fun create(modelClass: Class): T { + return InteractUiElementViewModel(useCase, resourceProvider) as T + } + } +} + +data class SelectedUiElementState( + val description: String, + val packageName: String, + val appName: String, + val appIcon: ComposeIconInfo.Drawable?, + val nodeText: String?, + val nodeClassName: String?, + val nodeViewResourceId: String?, + val nodeUniqueId: String?, + val interactionTypes: List>, + val selectedInteraction: NodeInteractionType, +) + +sealed class RecordUiElementState { + data class Recorded(val interactionCount: Int) : RecordUiElementState() + + data class CountingDown( + val timeRemaining: String, + val interactionCount: Int, + ) : RecordUiElementState() + + data object Empty : RecordUiElementState() +} + +data class SelectUiElementState( + val listItems: List, + val interactionTypes: List>, + val selectedInteractionType: NodeInteractionType?, +) + +data class UiElementListItemModel( + val id: Long, + val nodeViewResourceId: String?, + val nodeText: String?, + val nodeClassName: String?, + val nodeUniqueId: String?, + val interactionTypesText: String, + val interactionTypes: Set, +) diff --git a/app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt new file mode 100644 index 0000000000..f31a6520f5 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/actions/uielement/NodeInteractionType.kt @@ -0,0 +1,14 @@ +package io.github.sds100.keymapper.actions.uielement + +import android.view.accessibility.AccessibilityNodeInfo + +enum class NodeInteractionType(val accessibilityActionId: Int) { + CLICK(AccessibilityNodeInfo.ACTION_CLICK), + LONG_CLICK(AccessibilityNodeInfo.ACTION_LONG_CLICK), + FOCUS(AccessibilityNodeInfo.ACTION_FOCUS), + SELECT(AccessibilityNodeInfo.ACTION_SELECT), + SCROLL_FORWARD(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD), + SCROLL_BACKWARD(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD), + EXPAND(AccessibilityNodeInfo.ACTION_EXPAND), + COLLAPSE(AccessibilityNodeInfo.ACTION_COLLAPSE), +} diff --git a/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt b/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt index d9664aaba6..07a71956d1 100644 --- a/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt +++ b/app/src/main/java/io/github/sds100/keymapper/backup/BackupContent.kt @@ -6,7 +6,6 @@ import io.github.sds100.keymapper.data.entities.FloatingLayoutEntity import io.github.sds100.keymapper.data.entities.GroupEntity import io.github.sds100.keymapper.data.entities.KeyMapEntity -// TODO back up groups that are referenced by key maps - back up all the children as well. If the parent is not included in the back up then set the parent uid to null data class BackupContent( @SerializedName(NAME_DB_VERSION) val dbVersion: Int, diff --git a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt index 659c66b79e..f2e22b2ce8 100644 --- a/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt +++ b/app/src/main/java/io/github/sds100/keymapper/backup/BackupManager.kt @@ -249,6 +249,9 @@ class BackupManagerImpl( // Do nothing. It just removed the group name index. JsonMigration(17, 18) { json -> json }, + + // Do nothing. Just added the accessibility node table. + JsonMigration(18, 19) { json -> json }, ) if (keyMapListJsonArray != null) { diff --git a/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt b/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt index 0c7f2678ae..b015752aca 100644 --- a/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt +++ b/app/src/main/java/io/github/sds100/keymapper/constraints/ConstraintId.kt @@ -49,5 +49,5 @@ enum class ConstraintId { CHARGING, DISCHARGING, - TIME + TIME, } diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt index e3a933e747..e4644b1055 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/AppDatabase.kt @@ -9,6 +9,7 @@ import androidx.room.TypeConverters import androidx.room.migration.Migration import androidx.sqlite.db.SupportSQLiteDatabase import io.github.sds100.keymapper.data.db.AppDatabase.Companion.DATABASE_VERSION +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao import io.github.sds100.keymapper.data.db.dao.FingerprintMapDao import io.github.sds100.keymapper.data.db.dao.FloatingButtonDao import io.github.sds100.keymapper.data.db.dao.FloatingLayoutDao @@ -18,7 +19,9 @@ import io.github.sds100.keymapper.data.db.dao.LogEntryDao import io.github.sds100.keymapper.data.db.typeconverter.ActionListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.ConstraintListTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.ExtraListTypeConverter +import io.github.sds100.keymapper.data.db.typeconverter.NodeInteractionTypeSetTypeConverter import io.github.sds100.keymapper.data.db.typeconverter.TriggerTypeConverter +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity import io.github.sds100.keymapper.data.entities.FingerprintMapEntity import io.github.sds100.keymapper.data.entities.FloatingButtonEntity import io.github.sds100.keymapper.data.entities.FloatingLayoutEntity @@ -28,6 +31,7 @@ import io.github.sds100.keymapper.data.entities.LogEntryEntity import io.github.sds100.keymapper.data.migration.AutoMigration14To15 import io.github.sds100.keymapper.data.migration.AutoMigration15To16 import io.github.sds100.keymapper.data.migration.AutoMigration16To17 +import io.github.sds100.keymapper.data.migration.AutoMigration18To19 import io.github.sds100.keymapper.data.migration.Migration10To11 import io.github.sds100.keymapper.data.migration.Migration11To12 import io.github.sds100.keymapper.data.migration.Migration13To14 @@ -44,7 +48,7 @@ import io.github.sds100.keymapper.data.migration.Migration9To10 * Created by sds100 on 24/01/2020. */ @Database( - entities = [KeyMapEntity::class, FingerprintMapEntity::class, LogEntryEntity::class, FloatingLayoutEntity::class, FloatingButtonEntity::class, GroupEntity::class], + entities = [KeyMapEntity::class, FingerprintMapEntity::class, LogEntryEntity::class, FloatingLayoutEntity::class, FloatingButtonEntity::class, GroupEntity::class, AccessibilityNodeEntity::class], version = DATABASE_VERSION, exportSchema = true, autoMigrations = [ @@ -54,6 +58,8 @@ import io.github.sds100.keymapper.data.migration.Migration9To10 AutoMigration(from = 15, to = 16, spec = AutoMigration15To16::class), // This adds last opened timestamp to groups AutoMigration(from = 16, to = 17, spec = AutoMigration16To17::class), + // Adds accessibility node table + AutoMigration(from = 18, to = 19, spec = AutoMigration18To19::class), ], ) @TypeConverters( @@ -61,11 +67,12 @@ import io.github.sds100.keymapper.data.migration.Migration9To10 ExtraListTypeConverter::class, TriggerTypeConverter::class, ConstraintListTypeConverter::class, + NodeInteractionTypeSetTypeConverter::class, ) abstract class AppDatabase : RoomDatabase() { companion object { const val DATABASE_NAME = "key_map_database" - const val DATABASE_VERSION = 18 + const val DATABASE_VERSION = 19 val MIGRATION_1_2 = object : Migration(1, 2) { @@ -162,4 +169,5 @@ abstract class AppDatabase : RoomDatabase() { abstract fun floatingLayoutDao(): FloatingLayoutDao abstract fun floatingButtonDao(): FloatingButtonDao abstract fun groupDao(): GroupDao + abstract fun accessibilityNodeDao(): AccessibilityNodeDao } diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt new file mode 100644 index 0000000000..ed2bd36250 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/dao/AccessibilityNodeDao.kt @@ -0,0 +1,35 @@ +package io.github.sds100.keymapper.data.db.dao + +import androidx.room.Dao +import androidx.room.Insert +import androidx.room.OnConflictStrategy +import androidx.room.Query +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import kotlinx.coroutines.flow.Flow + +@Dao +interface AccessibilityNodeDao { + companion object { + const val TABLE_NAME = "accessibility_nodes" + const val KEY_ID = "id" + const val KEY_PACKAGE_NAME = "package_name" + const val KEY_TEXT = "text" + const val KEY_CONTENT_DESCRIPTION = "content_description" + const val KEY_CLASS_NAME = "class_name" + const val KEY_VIEW_RESOURCE_ID = "view_resource_id" + const val KEY_UNIQUE_ID = "unique_id" + const val KEY_ACTIONS = "actions" + } + + @Query("SELECT * FROM $TABLE_NAME WHERE $KEY_ID = (:id)") + suspend fun getById(id: Long): AccessibilityNodeEntity? + + @Query("SELECT * FROM $TABLE_NAME") + fun getAll(): Flow> + + @Insert(onConflict = OnConflictStrategy.ABORT) + suspend fun insert(vararg node: AccessibilityNodeEntity) + + @Query("DELETE FROM $TABLE_NAME") + suspend fun deleteAll() +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt b/app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt new file mode 100644 index 0000000000..d4093d82d8 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/db/typeconverter/NodeInteractionTypeSetTypeConverter.kt @@ -0,0 +1,34 @@ +package io.github.sds100.keymapper.data.db.typeconverter + +import androidx.room.TypeConverter +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType + +/** + * Created by sds100 on 05/09/2018. + */ + +class NodeInteractionTypeSetTypeConverter { + @TypeConverter + fun toSet(mask: Int): Set { + val interactionTypeSet = mutableSetOf() + + for (type in NodeInteractionType.entries) { + if (mask and type.accessibilityActionId == type.accessibilityActionId) { + interactionTypeSet.add(type) + } + } + + return interactionTypeSet + } + + @TypeConverter + fun toMask(set: Set): Int { + var nodeActionMask = 0 + + for (nodeAction in set) { + nodeActionMask = nodeActionMask or nodeAction.accessibilityActionId + } + + return nodeActionMask + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt new file mode 100644 index 0000000000..ca3ba0ee1c --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/AccessibilityNodeEntity.kt @@ -0,0 +1,48 @@ +package io.github.sds100.keymapper.data.entities + +import android.os.Parcelable +import androidx.room.ColumnInfo +import androidx.room.Entity +import androidx.room.PrimaryKey +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_ACTIONS +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_CLASS_NAME +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_CONTENT_DESCRIPTION +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_ID +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_PACKAGE_NAME +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_TEXT +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_UNIQUE_ID +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.KEY_VIEW_RESOURCE_ID +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao.Companion.TABLE_NAME +import kotlinx.parcelize.Parcelize +import kotlinx.serialization.Serializable + +@Serializable +@Parcelize +@Entity(tableName = TABLE_NAME) +data class AccessibilityNodeEntity( + @PrimaryKey(autoGenerate = true) + @ColumnInfo(name = KEY_ID) + val id: Long = 0L, + + @ColumnInfo(name = KEY_PACKAGE_NAME) + val packageName: String, + + @ColumnInfo(name = KEY_TEXT) + val text: String?, + + @ColumnInfo(name = KEY_CONTENT_DESCRIPTION) + val contentDescription: String?, + + @ColumnInfo(name = KEY_CLASS_NAME) + val className: String?, + + @ColumnInfo(name = KEY_VIEW_RESOURCE_ID) + val viewResourceId: String?, + + @ColumnInfo(name = KEY_UNIQUE_ID) + val uniqueId: String?, + + @ColumnInfo(name = KEY_ACTIONS) + val actions: Set, +) : Parcelable diff --git a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt index e99db4de73..4385e0b3a4 100644 --- a/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt +++ b/app/src/main/java/io/github/sds100/keymapper/data/entities/ActionEntity.kt @@ -90,6 +90,17 @@ data class ActionEntity( const val EXTRA_HTTP_DESCRIPTION = "extra_http_description" const val EXTRA_HTTP_AUTHORIZATION_HEADER = "extra_http_authorization_header" + // Accessibility node extras + const val EXTRA_ACCESSIBILITY_PACKAGE_NAME = "extra_accessibility_package_name" + const val EXTRA_ACCESSIBILITY_CONTENT_DESCRIPTION = + "extra_accessibility_content_description" + const val EXTRA_ACCESSIBILITY_TEXT = "extra_accessibility_text" + const val EXTRA_ACCESSIBILITY_CLASS_NAME = "extra_accessibility_class_name" + const val EXTRA_ACCESSIBILITY_VIEW_RESOURCE_ID = "extra_accessibility_view_resource_id" + const val EXTRA_ACCESSIBILITY_UNIQUE_ID = "extra_accessibility_unique_id" + const val EXTRA_ACCESSIBILITY_ACTIONS = "extra_accessibility_actions" + const val EXTRA_ACCESSIBILITY_NODE_ACTION = "extra_accessibility_node_action" + // DON'T CHANGE THESE. Used for JSON serialization and parsing. const val NAME_ACTION_TYPE = "type" const val NAME_DATA = "data" @@ -153,6 +164,7 @@ data class ActionEntity( INTENT, PHONE_CALL, SOUND, + INTERACT_UI_ELEMENT, } constructor( diff --git a/app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt b/app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt new file mode 100644 index 0000000000..6ee2b50840 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/migration/AutoMigration18To19.kt @@ -0,0 +1,5 @@ +package io.github.sds100.keymapper.data.migration + +import androidx.room.migration.AutoMigrationSpec + +class AutoMigration18To19 : AutoMigrationSpec diff --git a/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt new file mode 100644 index 0000000000..8783d66dd5 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/data/repositories/AccessibilityNodeRepository.kt @@ -0,0 +1,60 @@ +package io.github.sds100.keymapper.data.repositories + +import android.database.sqlite.SQLiteConstraintException +import io.github.sds100.keymapper.data.db.dao.AccessibilityNodeDao +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.util.State +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +interface AccessibilityNodeRepository { + val nodes: Flow>> + suspend fun get(id: Long): AccessibilityNodeEntity? + fun insert(vararg node: AccessibilityNodeEntity) + suspend fun deleteAll() +} + +class AccessibilityNodeRepositoryImpl( + private val coroutineScope: CoroutineScope, + private val dao: AccessibilityNodeDao, +) : AccessibilityNodeRepository { + + override val nodes: StateFlow>> = + dao.getAll() + .map { list -> + // Distinct by all fields except the ID. + State.Data(list.distinctBy { it.copy(id = 0) }) + } + .flowOn(Dispatchers.IO) + .stateIn(coroutineScope, SharingStarted.WhileSubscribed(10000), State.Loading) + + override fun insert(vararg node: AccessibilityNodeEntity) { + coroutineScope.launch(Dispatchers.IO) { + for (n in node) { + try { + dao.insert(n) + } catch (e: SQLiteConstraintException) { + // Do nothing if the node already exists. + } + } + } + } + + override suspend fun get(id: Long): AccessibilityNodeEntity? { + return dao.getById(id) + } + + override suspend fun deleteAll() { + withContext(Dispatchers.IO) { + dao.deleteAll() + } + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt deleted file mode 100644 index 3e0675d83d..0000000000 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityEventModel.kt +++ /dev/null @@ -1,6 +0,0 @@ -package io.github.sds100.keymapper.system.accessibility - -/** - * Created by sds100 on 27/07/2021. - */ -data class AccessibilityEventModel(val eventTime: Long, val eventType: Int) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt index 934c0ace2f..08f5a14e04 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeModel.kt @@ -1,8 +1,13 @@ package io.github.sds100.keymapper.system.accessibility +import android.os.Build +import androidx.annotation.RequiresApi +import kotlinx.serialization.Serializable + /** * Created by sds100 on 21/04/2021. */ +@Serializable data class AccessibilityNodeModel( val packageName: String?, val contentDescription: String?, @@ -11,4 +16,14 @@ data class AccessibilityNodeModel( val textSelectionStart: Int, val textSelectionEnd: Int, val isEditable: Boolean, + val className: String?, + val viewResourceId: String?, + + @RequiresApi(Build.VERSION_CODES.TIRAMISU) + val uniqueId: String?, + + /** + * A list of the allowed accessibility node actions. + */ + val actions: List, ) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt new file mode 100644 index 0000000000..b0e1250a56 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityNodeRecorder.kt @@ -0,0 +1,137 @@ +package io.github.sds100.keymapper.system.accessibility + +import android.graphics.Rect +import android.os.Build +import android.os.CountDownTimer +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import io.github.sds100.keymapper.actions.uielement.NodeInteractionType +import io.github.sds100.keymapper.data.entities.AccessibilityNodeEntity +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update + +class AccessibilityNodeRecorder( + private val nodeRepository: AccessibilityNodeRepository, +) { + companion object { + private const val RECORD_DURATION = 60000L + } + + private val timerLock = Any() + private var timer: CountDownTimer? = null + private val _recordState: MutableStateFlow = + MutableStateFlow(RecordAccessibilityNodeState.Idle) + val recordState = _recordState.asStateFlow() + + fun startRecording() { + synchronized(timerLock) { + timer?.cancel() + timer = object : CountDownTimer(RECORD_DURATION, 1000) { + + override fun onTick(millisUntilFinished: Long) { + _recordState.update { + RecordAccessibilityNodeState.CountingDown( + timeLeft = (millisUntilFinished / 1000).toInt(), + ) + } + } + + override fun onFinish() { + _recordState.update { RecordAccessibilityNodeState.Idle } + } + } + + timer!!.start() + } + } + + fun stopRecording() { + synchronized(timerLock) { + timer?.cancel() + timer = null + _recordState.update { RecordAccessibilityNodeState.Idle } + } + } + + fun onAccessibilityEvent(event: AccessibilityEvent) { + if (_recordState.value is RecordAccessibilityNodeState.Idle) { + return + } + + val source = event.source ?: return + val sourceBounds = Rect() + source.getBoundsInScreen(sourceBounds) + + val root: AccessibilityNodeInfo = source.window.root ?: return + + // This searches for all nodes that are within the bounds of the source of the + // AccessibilityEvent because the source is not necessarily the element + // the user wants to tap. + val entities = getNodesInBounds(root, sourceBounds).toTypedArray() + nodeRepository.insert(*entities) + } + + /** + * Get all the nodes that are within the given bounds. + */ + private fun getNodesInBounds( + node: AccessibilityNodeInfo, + bounds: Rect, + ): Set { + val set = mutableSetOf() + + val nodeBounds = Rect() + node.getBoundsInScreen(nodeBounds) + + if (bounds.contains(nodeBounds)) { + val entity = buildNodeEntity(node) + + if (entity != null) { + set.add(entity) + } + } + + if (node.childCount > 0) { + for (i in 0 until node.childCount) { + val child = node.getChild(i) ?: continue + + set.addAll(getNodesInBounds(child, bounds)) + } + } + + return set + } + + private fun buildNodeEntity(source: AccessibilityNodeInfo): AccessibilityNodeEntity? { + val interactionTypes = source.actionList.mapNotNull { action -> + NodeInteractionType.entries.find { it.accessibilityActionId == action.id } + }.distinct() + + if (interactionTypes.isEmpty()) { + return null + } + + return AccessibilityNodeEntity( + packageName = source.packageName.toString(), + text = source.text?.toString(), + contentDescription = source.contentDescription?.toString(), + className = source.className?.toString(), + viewResourceId = source.viewIdResourceName, + uniqueId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + source.uniqueId + } else { + null + }, + actions = interactionTypes.toSet(), + ) + } + + fun teardown() { + synchronized(timerLock) { + timer?.cancel() + timer = null + } + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt index 1f16e4da93..773486c357 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/AccessibilityUtils.kt @@ -1,6 +1,6 @@ package io.github.sds100.keymapper.system.accessibility -import android.view.accessibility.AccessibilityEvent +import android.os.Build import android.view.accessibility.AccessibilityNodeInfo /** @@ -38,6 +38,12 @@ fun AccessibilityNodeInfo.toModel(): AccessibilityNodeModel = AccessibilityNodeM textSelectionEnd = textSelectionEnd, text = text?.toString(), isEditable = isEditable, + className = className?.toString(), + uniqueId = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + uniqueId + } else { + null + }, + viewResourceId = viewIdResourceName, + actions = actionList.map { it.id }, ) - -fun AccessibilityEvent.toModel(): AccessibilityEventModel = AccessibilityEventModel(eventTime, eventType) diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt index f29c038ee8..d0a3347b75 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/BaseAccessibilityServiceController.kt @@ -11,6 +11,7 @@ import io.github.sds100.keymapper.actions.PerformActionsUseCase import io.github.sds100.keymapper.constraints.DetectConstraintsUseCase import io.github.sds100.keymapper.data.Keys import io.github.sds100.keymapper.data.PreferenceDefaults +import io.github.sds100.keymapper.data.repositories.AccessibilityNodeRepository import io.github.sds100.keymapper.data.repositories.PreferenceRepository import io.github.sds100.keymapper.mappings.FingerprintGestureType import io.github.sds100.keymapper.mappings.FingerprintGesturesSupportedUseCase @@ -40,6 +41,8 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharedFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.drop @@ -47,6 +50,7 @@ import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.flow.update import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -60,7 +64,7 @@ import timber.log.Timber */ abstract class BaseAccessibilityServiceController( private val coroutineScope: CoroutineScope, - private val accessibilityService: IAccessibilityService, + private val service: MyAccessibilityService, private val inputEvents: SharedFlow, private val outputEvents: MutableSharedFlow, private val detectConstraintsUseCase: DetectConstraintsUseCase, @@ -73,6 +77,7 @@ abstract class BaseAccessibilityServiceController( private val suAdapter: SuAdapter, private val inputMethodAdapter: InputMethodAdapter, private val settingsRepository: PreferenceRepository, + private val nodeRepository: AccessibilityNodeRepository, ) { companion object { @@ -101,6 +106,9 @@ abstract class BaseAccessibilityServiceController( rerouteKeyEventsUseCase, ) + private val accessibilityNodeRecorder: AccessibilityNodeRecorder = + AccessibilityNodeRecorder(nodeRepository) + private var recordingTriggerJob: Job? = null private val recordingTrigger: Boolean get() = recordingTriggerJob != null && recordingTriggerJob?.isActive == true @@ -114,7 +122,7 @@ abstract class BaseAccessibilityServiceController( detectKeyMapsUseCase.detectScreenOffTriggers .stateIn(coroutineScope, SharingStarted.Eagerly, false) - private val changeImeOnInputFocus: StateFlow = + private val changeImeOnInputFocusFlow: StateFlow = settingsRepository .get(Keys.changeImeOnInputFocus) .map { it ?: PreferenceDefaults.CHANGE_IME_ON_INPUT_FOCUS } @@ -145,6 +153,7 @@ abstract class BaseAccessibilityServiceController( // This is required for receive TYPE_WINDOWS_CHANGED events so can // detect when to show/hide overlays. .withFlag(AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS) + .withFlag(AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { flags = flags.withFlag(AccessibilityServiceInfo.FLAG_ENABLE_ACCESSIBILITY_VOLUME) @@ -171,24 +180,25 @@ abstract class BaseAccessibilityServiceController( MutableStateFlow(AccessibilityEvent.TYPE_WINDOWS_CHANGED) init { + serviceFlags.onEach { flags -> // check that it isn't null because this can only be called once the service is bound - if (accessibilityService.serviceFlags != null) { - accessibilityService.serviceFlags = flags + if (service.serviceFlags != null) { + service.serviceFlags = flags } }.launchIn(coroutineScope) serviceFeedbackType.onEach { feedbackType -> // check that it isn't null because this can only be called once the service is bound - if (accessibilityService.serviceFeedbackType != null) { - accessibilityService.serviceFeedbackType = feedbackType + if (service.serviceFeedbackType != null) { + service.serviceFeedbackType = feedbackType } }.launchIn(coroutineScope) serviceEventTypes.onEach { eventTypes -> // check that it isn't null because this can only be called once the service is bound - if (accessibilityService.serviceEventTypes != null) { - accessibilityService.serviceEventTypes = eventTypes + if (service.serviceEventTypes != null) { + service.serviceEventTypes = eventTypes } }.launchIn(coroutineScope) @@ -224,7 +234,7 @@ abstract class BaseAccessibilityServiceController( onEventFromUi(it) }.launchIn(coroutineScope) - accessibilityService.isKeyboardHidden + service.isKeyboardHidden .drop(1) // Don't send it when collecting initially .onEach { isHidden -> if (isHidden) { @@ -255,23 +265,53 @@ abstract class BaseAccessibilityServiceController( } }.launchIn(coroutineScope) - changeImeOnInputFocus.onEach { changeImeOnInputFocus -> - if (changeImeOnInputFocus) { - serviceEventTypes.value = serviceEventTypes.value - .withFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) - .withFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) - } else { - serviceEventTypes.value = serviceEventTypes.value - .minusFlag(AccessibilityEvent.TYPE_VIEW_FOCUSED) - .minusFlag(AccessibilityEvent.TYPE_VIEW_CLICKED) + coroutineScope.launch { + accessibilityNodeRecorder.recordState.collectLatest { state -> + outputEvents.emit(ServiceEvent.OnRecordNodeStateChanged(state)) } - }.launchIn(coroutineScope) + } + + val imeInputFocusEvents = + AccessibilityEvent.TYPE_VIEW_FOCUSED or AccessibilityEvent.TYPE_VIEW_CLICKED + + val recordNodeEvents = AccessibilityEvent.TYPE_VIEW_FOCUSED or + AccessibilityEvent.TYPE_VIEW_CLICKED or + AccessibilityEvent.TYPE_VIEW_LONG_CLICKED or + AccessibilityEvent.TYPE_VIEW_SELECTED or + AccessibilityEvent.TYPE_VIEW_SCROLLED + + coroutineScope.launch { + combine( + changeImeOnInputFocusFlow, + accessibilityNodeRecorder.recordState, + ) { changeImeOnInputFocus, recordState -> + + serviceEventTypes.update { eventTypes -> + var newEventTypes = eventTypes + + if (!changeImeOnInputFocus && recordState == RecordAccessibilityNodeState.Idle) { + newEventTypes = + newEventTypes and (imeInputFocusEvents or recordNodeEvents).inv() + } else { + if (changeImeOnInputFocus) { + newEventTypes = newEventTypes or imeInputFocusEvents + } + + if (recordState is RecordAccessibilityNodeState.CountingDown) { + newEventTypes = newEventTypes or recordNodeEvents + } + } + + newEventTypes + } + }.collect() + } } open fun onServiceConnected() { - accessibilityService.serviceFlags = serviceFlags.value - accessibilityService.serviceFeedbackType = serviceFeedbackType.value - accessibilityService.serviceEventTypes = serviceEventTypes.value + service.serviceFlags = serviceFlags.value + service.serviceFeedbackType = serviceFeedbackType.value + service.serviceEventTypes = serviceEventTypes.value // check if fingerprint gestures are supported if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { @@ -284,7 +324,7 @@ abstract class BaseAccessibilityServiceController( * used while this is called. */ if (fingerprintGesturesSupported.isSupported.firstBlocking() != true) { fingerprintGesturesSupported.setSupported( - accessibilityService.isFingerprintGestureDetectionAvailable, + service.isFingerprintGestureDetectionAvailable, ) } @@ -294,6 +334,10 @@ abstract class BaseAccessibilityServiceController( } } + open fun onDestroy() { + accessibilityNodeRecorder.teardown() + } + open fun onConfigurationChanged(newConfig: Configuration) { } @@ -441,10 +485,12 @@ abstract class BaseAccessibilityServiceController( } } - open fun onAccessibilityEvent(event: AccessibilityEventModel) { - if (changeImeOnInputFocus.value) { + open fun onAccessibilityEvent(event: AccessibilityEvent) { + accessibilityNodeRecorder.onAccessibilityEvent(event) + + if (changeImeOnInputFocusFlow.value) { val focussedNode = - accessibilityService.findFocussedNode(AccessibilityNodeInfo.FOCUS_INPUT) + service.findFocussedNode(AccessibilityNodeInfo.FOCUS_INPUT) if (focussedNode?.isEditable == true && focussedNode.isFocused) { Timber.d("Got input focus") @@ -501,17 +547,25 @@ abstract class BaseAccessibilityServiceController( outputEvents.emit(ServiceEvent.Pong(event.key)) } - is ServiceEvent.HideKeyboard -> accessibilityService.hideKeyboard() - is ServiceEvent.ShowKeyboard -> accessibilityService.showKeyboard() - is ServiceEvent.ChangeIme -> accessibilityService.switchIme(event.imeId) + is ServiceEvent.HideKeyboard -> service.hideKeyboard() + is ServiceEvent.ShowKeyboard -> service.showKeyboard() + is ServiceEvent.ChangeIme -> service.switchIme(event.imeId) is ServiceEvent.DisableService -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - accessibilityService.disableSelf() + service.disableSelf() } is ServiceEvent.TriggerKeyMap -> triggerKeyMapFromIntent(event.uid) is ServiceEvent.EnableInputMethod -> if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - accessibilityService.setInputMethodEnabled(event.imeId, true) + service.setInputMethodEnabled(event.imeId, true) + } + + is ServiceEvent.StartRecordingNodes -> { + accessibilityNodeRecorder.startRecording() + } + + is ServiceEvent.StopRecordingNodes -> { + accessibilityNodeRecorder.stopRecording() } else -> Unit diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt index 2fdae08f4f..99205867b6 100644 --- a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/MyAccessibilityService.kt @@ -22,7 +22,9 @@ import androidx.lifecycle.LifecycleRegistry import androidx.savedstate.SavedStateRegistry import androidx.savedstate.SavedStateRegistryController import androidx.savedstate.SavedStateRegistryOwner +import io.github.sds100.keymapper.Constants import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.ServiceLocator import io.github.sds100.keymapper.actions.pinchscreen.PinchScreenType import io.github.sds100.keymapper.api.IKeyEventRelayServiceCallback import io.github.sds100.keymapper.api.KeyEventRelayService @@ -38,6 +40,7 @@ import io.github.sds100.keymapper.util.InputEventType import io.github.sds100.keymapper.util.MathUtils import io.github.sds100.keymapper.util.Result import io.github.sds100.keymapper.util.Success +import io.github.sds100.keymapper.util.onSuccess import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.update @@ -188,6 +191,14 @@ class MyAccessibilityService : override fun onServiceConnected() { super.onServiceConnected() + val inputMethodAdapter = ServiceLocator.inputMethodAdapter(this) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + inputMethodAdapter.getInfoByPackageName(Constants.PACKAGE_NAME).onSuccess { + softKeyboardController.setInputMethodEnabled(it.id, true) + softKeyboardController.switchToInputMethod(it.id) + } + } Timber.i("Accessibility service: onServiceConnected") lifecycleRegistry.currentState = Lifecycle.State.STARTED @@ -244,6 +255,7 @@ class MyAccessibilityService : override fun onInterrupt() {} override fun onDestroy() { + controller?.onDestroy() controller = null lifecycleRegistry.currentState = Lifecycle.State.DESTROYED @@ -282,7 +294,7 @@ class MyAccessibilityService : _activeWindowPackage.update { rootInActiveWindow?.packageName?.toString() } } - controller?.onAccessibilityEvent(event.toModel()) + controller?.onAccessibilityEvent(event) } override fun onKeyEvent(event: KeyEvent?): Boolean { diff --git a/app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt new file mode 100644 index 0000000000..ea1e26d091 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/accessibility/RecordAccessibilityNodeState.kt @@ -0,0 +1,15 @@ +package io.github.sds100.keymapper.system.accessibility + +import kotlinx.serialization.Serializable + +@Serializable +sealed class RecordAccessibilityNodeState { + data object Idle : RecordAccessibilityNodeState() + + data class CountingDown( + /** + * The time left in seconds + */ + val timeLeft: Int, + ) : RecordAccessibilityNodeState() +} diff --git a/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt b/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt new file mode 100644 index 0000000000..ca05260be4 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/system/apps/ChooseAppScreen.kt @@ -0,0 +1,216 @@ +package io.github.sds100.keymapper.system.apps + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.calculateEndPadding +import androidx.compose.foundation.layout.calculateStartPadding +import androidx.compose.foundation.layout.displayCutoutPadding +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material3.BottomAppBar +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalLayoutDirection +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import io.github.sds100.keymapper.R +import io.github.sds100.keymapper.compose.KeyMapperTheme +import io.github.sds100.keymapper.util.State +import io.github.sds100.keymapper.util.drawable +import io.github.sds100.keymapper.util.ui.compose.ComposeIconInfo +import io.github.sds100.keymapper.util.ui.compose.SearchAppBarActions +import io.github.sds100.keymapper.util.ui.compose.SimpleListItem +import io.github.sds100.keymapper.util.ui.compose.SimpleListItemModel + +@Composable +fun ChooseAppScreen( + modifier: Modifier = Modifier, + title: String, + state: State>, + query: String? = null, + onQueryChange: (String) -> Unit = {}, + onCloseSearch: () -> Unit = {}, + onNavigateBack: () -> Unit = {}, + onClickApp: (String) -> Unit = {}, +) { + Scaffold( + modifier.displayCutoutPadding(), + bottomBar = { + BottomAppBar( + modifier = Modifier.imePadding(), + actions = { + SearchAppBarActions( + onCloseSearch = onCloseSearch, + onNavigateBack = onNavigateBack, + onQueryChange = onQueryChange, + enabled = state is State.Data, + query = query, + ) + }, + ) + }, + ) { innerPadding -> + val layoutDirection = LocalLayoutDirection.current + val startPadding = innerPadding.calculateStartPadding(layoutDirection) + val endPadding = innerPadding.calculateEndPadding(layoutDirection) + + Surface( + modifier = Modifier + .fillMaxSize() + .padding( + top = innerPadding.calculateTopPadding(), + bottom = innerPadding.calculateBottomPadding(), + start = startPadding, + end = endPadding, + ), + ) { + Column { + Text( + modifier = Modifier.padding( + start = 16.dp, + end = 16.dp, + top = 16.dp, + bottom = 8.dp, + ), + text = title, + style = MaterialTheme.typography.titleLarge, + ) + + Spacer(modifier = Modifier.height(8.dp)) + + when (state) { + State.Loading -> LoadingScreen(modifier = Modifier.fillMaxSize()) + is State.Data -> { + val items = state.data + + if (items.isEmpty()) { + EmptyScreen(modifier = Modifier.fillMaxSize()) + } else { + ListScreen( + modifier = Modifier.fillMaxSize(), + listItems = items, + onClick = onClickApp, + ) + } + } + } + } + } + } +} + +@Composable +private fun LoadingScreen(modifier: Modifier = Modifier) { + Box(modifier) { + CircularProgressIndicator(modifier = Modifier.align(Alignment.Center)) + } +} + +@Composable +private fun EmptyScreen(modifier: Modifier = Modifier) { + Box(modifier) { + val shrug = stringResource(R.string.shrug) + val text = stringResource(R.string.app_list_empty) + Text( + modifier = Modifier.align(Alignment.Center), + text = buildAnnotatedString { + withStyle(MaterialTheme.typography.headlineLarge.toSpanStyle()) { + append(shrug) + } + appendLine() + appendLine() + withStyle(MaterialTheme.typography.bodyLarge.toSpanStyle()) { + append(text) + } + }, + textAlign = TextAlign.Center, + ) + } +} + +@Composable +private fun ListScreen( + modifier: Modifier = Modifier, + listItems: List, + onClick: (String) -> Unit, +) { + LazyColumn( + modifier = modifier, + contentPadding = PaddingValues(8.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + items(listItems) { model -> + SimpleListItem( + modifier = Modifier.fillMaxWidth(), + model = model, + onClick = { onClick(model.id) }, + ) + } + } +} + +@Preview +@Composable +private fun Empty() { + KeyMapperTheme { + ChooseAppScreen(title = "Choose app", state = State.Data(emptyList())) + } +} + +@Preview +@Composable +private fun Loading() { + KeyMapperTheme { + ChooseAppScreen(title = "Choose app", state = State.Loading) + } +} + +@Preview +@Composable +private fun Loaded() { + val icon = LocalContext.current.drawable(R.mipmap.ic_launcher_round) + + KeyMapperTheme { + ChooseAppScreen( + title = "Choose app", + state = State.Data( + listOf( + SimpleListItemModel( + id = "1", + title = "Key Mapper", + icon = ComposeIconInfo.Drawable(icon), + ), + SimpleListItemModel( + id = "2", + title = "Key Mapper", + icon = ComposeIconInfo.Drawable(icon), + ), + SimpleListItemModel( + id = "3", + title = "Key Mapper", + icon = ComposeIconInfo.Drawable(icon), + ), + ), + ), + ) + } +} diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt b/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt index 4aa7e57171..8a047e25b0 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ErrorUtils.kt @@ -162,6 +162,7 @@ fun Error.getFullMessage(resourceProvider: ResourceProvider): String = when (thi Error.DpadTriggerImeNotSelected -> resourceProvider.getString(R.string.trigger_error_dpad_ime_not_selected) Error.InvalidBackup -> resourceProvider.getString(R.string.error_invalid_backup) Error.MalformedUrl -> resourceProvider.getString(R.string.error_malformed_url) + Error.UiElementNotFound -> resourceProvider.getString(R.string.error_ui_element_not_found) } val Error.isFixable: Boolean diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt index 41a4f69d47..6bb6786364 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Inject.kt @@ -15,6 +15,7 @@ import io.github.sds100.keymapper.actions.sound.ChooseSoundFileUseCaseImpl import io.github.sds100.keymapper.actions.sound.ChooseSoundFileViewModel import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateViewModel import io.github.sds100.keymapper.actions.tapscreen.PickDisplayCoordinateViewModel +import io.github.sds100.keymapper.actions.uielement.InteractUiElementViewModel import io.github.sds100.keymapper.api.KeyEventRelayServiceWrapper import io.github.sds100.keymapper.backup.BackupRestoreMappingsUseCaseImpl import io.github.sds100.keymapper.constraints.ChooseConstraintViewModel @@ -229,6 +230,7 @@ object Inject { ), inputMethodAdapter = ServiceLocator.inputMethodAdapter(service), settingsRepository = ServiceLocator.settingsRepository(service), + nodeRepository = ServiceLocator.accessibilityNodeRepository(service), ) fun chooseBluetoothDeviceViewModel(ctx: Context): ChooseBluetoothDeviceViewModel.Factory = ChooseBluetoothDeviceViewModel.Factory( @@ -248,4 +250,11 @@ object Inject { ), ServiceLocator.resourceProvider(ctx), ) + + fun interactUiElementViewModel( + ctx: Context, + ): InteractUiElementViewModel.Factory = InteractUiElementViewModel.Factory( + (ctx.applicationContext as KeyMapperApp).interactUiElementController, + resourceProvider = ServiceLocator.resourceProvider(ctx), + ) } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/Result.kt b/app/src/main/java/io/github/sds100/keymapper/util/Result.kt index 0d904a6685..8fe8251769 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/Result.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/Result.kt @@ -145,6 +145,8 @@ sealed class Error : Result() { */ data object DpadTriggerImeNotSelected : Error() data object MalformedUrl : Error() + + data object UiElementNotFound : Error() } inline fun Result.onSuccess(f: (T) -> Unit): Result { diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt b/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt index 054dd384fa..4bacac2040 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ServiceEvent.kt @@ -3,6 +3,7 @@ package io.github.sds100.keymapper.util import android.os.Parcelable import io.github.sds100.keymapper.actions.ActionData import io.github.sds100.keymapper.mappings.keymaps.trigger.KeyEventDetectionSource +import io.github.sds100.keymapper.system.accessibility.RecordAccessibilityNodeState import io.github.sds100.keymapper.system.devices.InputDeviceInfo import kotlinx.parcelize.Parcelize import kotlinx.serialization.Serializable @@ -72,4 +73,13 @@ sealed class ServiceEvent { @Serializable data class EnableInputMethod(val imeId: String) : ServiceEvent() + + @Serializable + data object StartRecordingNodes : ServiceEvent() + + @Serializable + data object StopRecordingNodes : ServiceEvent() + + @Serializable + data class OnRecordNodeStateChanged(val state: RecordAccessibilityNodeState) : ServiceEvent() } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt index 4e3ede84f8..de4cb1bea8 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavDestination.kt @@ -36,6 +36,7 @@ sealed class NavDestination { const val ID_CONFIG_KEY_MAP = "config_key_map" const val ID_SHIZUKU_SETTINGS = "shizuku_settings" const val ID_CONFIG_FLOATING_BUTTON = "config_floating_button" + const val ID_INTERACT_UI_ELEMENT_ACTION = "interact_ui_element_action" } data class ChooseApp( @@ -123,4 +124,8 @@ sealed class NavDestination { data class ConfigFloatingButton(val buttonUid: String?) : NavDestination() { override val id: String = ID_CONFIG_FLOATING_BUTTON } + + data class InteractUiElement(val action: ActionData.InteractUiElement?) : NavDestination() { + override val id: String = ID_INTERACT_UI_ELEMENT_ACTION + } } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt index eaf76d4b9d..fc9c239439 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/NavigationViewModel.kt @@ -20,6 +20,7 @@ import io.github.sds100.keymapper.actions.swipescreen.SwipePickCoordinateResult import io.github.sds100.keymapper.actions.swipescreen.SwipePickDisplayCoordinateFragment import io.github.sds100.keymapper.actions.tapscreen.PickCoordinateResult import io.github.sds100.keymapper.actions.tapscreen.PickDisplayCoordinateFragment +import io.github.sds100.keymapper.actions.uielement.InteractUiElementFragment import io.github.sds100.keymapper.constraints.ChooseConstraintFragment import io.github.sds100.keymapper.constraints.Constraint import io.github.sds100.keymapper.system.apps.ActivityInfo @@ -42,7 +43,6 @@ import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.merge import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.runBlocking -import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json /** @@ -225,6 +225,11 @@ fun NavigationViewModel.setupNavigation(fragment: Fragment) { is NavDestination.ConfigFloatingButton -> NavAppDirections.toConfigFloatingButton( destination.buttonUid, ) + + is NavDestination.InteractUiElement -> NavAppDirections.interactUiElement( + requestKey = requestKey, + action = destination.action?.let { Json.encodeToString(destination.action) }, + ) } fragment.findNavController().navigate(direction) @@ -326,5 +331,11 @@ fun NavigationViewModel.sendNavResultFromBundle( onNavResult(NavResult(requestKey, BluetoothDeviceInfo(address, name))) } + + NavDestination.ID_INTERACT_UI_ELEMENT_ACTION -> { + val json = bundle.getString(InteractUiElementFragment.EXTRA_ACTION)!! + val result = Json.decodeFromString(json) + onNavResult(NavResult(requestKey, result)) + } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt index f293b15968..eafba75c5d 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/KeyMapperDropdownMenu.kt @@ -10,18 +10,17 @@ import androidx.compose.material3.Text import androidx.compose.material3.TextField import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier -import androidx.compose.ui.res.stringResource -import io.github.sds100.keymapper.R -import io.github.sds100.keymapper.system.network.HttpMethod @Composable @OptIn(ExperimentalMaterial3Api::class) -fun KeyMapperDropdownMenu( +fun KeyMapperDropdownMenu( modifier: Modifier = Modifier, expanded: Boolean, onExpandedChange: (Boolean) -> Unit = {}, - value: String, - onValueChanged: (String) -> Unit = {}, + label: (@Composable () -> Unit)? = null, + selectedValue: T, + values: List>, + onValueChanged: (T) -> Unit = {}, ) { ExposedDropdownMenuBox( modifier = modifier, @@ -30,28 +29,31 @@ fun KeyMapperDropdownMenu( ) { TextField( modifier = Modifier.menuAnchor(MenuAnchorType.PrimaryNotEditable), - value = value, - onValueChange = onValueChanged, + value = values.find { it.first == selectedValue }?.second ?: values.first().second, + onValueChange = { newValue -> + onValueChanged(values.single { it.second == newValue }.first) + }, readOnly = true, - label = { Text(stringResource(R.string.action_http_request_method_label)) }, + label = label, trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded) }, colors = ExposedDropdownMenuDefaults.textFieldColors(), ) + ExposedDropdownMenu( matchTextFieldWidth = true, expanded = expanded, onDismissRequest = { onExpandedChange(false) }, ) { - for (method in HttpMethod.entries) { + for ((value, valueText) in values) { DropdownMenuItem( text = { Text( - method.toString(), + valueText, style = MaterialTheme.typography.bodyLarge, ) }, onClick = { - onValueChanged(method.toString()) + onValueChanged(value) onExpandedChange(false) }, contentPadding = ExposedDropdownMenuDefaults.ItemContentPadding, diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt new file mode 100644 index 0000000000..5ee078990d --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SearchAppBarActions.kt @@ -0,0 +1,90 @@ +package io.github.sds100.keymapper.util.ui.compose + +import androidx.compose.foundation.layout.RowScope +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.rounded.ArrowBack +import androidx.compose.material.icons.rounded.Search +import androidx.compose.material3.DockedSearchBar +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.SearchBarDefaults +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource +import io.github.sds100.keymapper.R + +@Composable +@OptIn(ExperimentalMaterial3Api::class) +fun RowScope.SearchAppBarActions( + onCloseSearch: () -> Unit, + onNavigateBack: () -> Unit, + onQueryChange: (String) -> Unit, + enabled: Boolean, + query: String?, +) { + var isExpanded: Boolean by rememberSaveable { mutableStateOf(false) } + + IconButton(onClick = { + if (isExpanded) { + onCloseSearch() + isExpanded = false + } else { + onNavigateBack() + } + }) { + Icon( + Icons.AutoMirrored.Rounded.ArrowBack, + contentDescription = stringResource(R.string.bottom_app_bar_back_content_description), + ) + } + + DockedSearchBar( + modifier = Modifier.Companion.align(Alignment.Companion.CenterVertically), + inputField = { + SearchBarDefaults.InputField( + modifier = Modifier.Companion.align(Alignment.Companion.CenterVertically), + onSearch = { + onQueryChange(it) + isExpanded = false + }, + leadingIcon = { + Icon( + Icons.Rounded.Search, + contentDescription = null, + ) + }, + enabled = enabled, + placeholder = { Text(stringResource(R.string.search_placeholder)) }, + query = query ?: "", + onQueryChange = onQueryChange, + expanded = isExpanded, + onExpandedChange = { expanded -> + if (expanded) { + isExpanded = true + } else { + onCloseSearch() + isExpanded = false + } + }, + ) + }, + // This is false to prevent an empty "content" showing underneath. + expanded = isExpanded, + onExpandedChange = { expanded -> + if (expanded) { + isExpanded = true + } else { + onCloseSearch() + isExpanded = false + } + }, + content = {}, + ) +} diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt index d3a31360bc..0c758c99e7 100644 --- a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/SimpleListItem.kt @@ -1,11 +1,14 @@ package io.github.sds100.keymapper.util.ui.compose +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.IntrinsicSize import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width @@ -13,11 +16,13 @@ import androidx.compose.material.icons.Icons import androidx.compose.material.icons.rounded.Wifi import androidx.compose.material3.Icon import androidx.compose.material3.LocalContentColor +import androidx.compose.material3.LocalMinimumInteractiveComponentSize import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedCard import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color @@ -46,58 +51,86 @@ fun SimpleListItemHeader( } @Composable -fun SimpleListItem( +fun SimpleListItemFixedHeight( modifier: Modifier = Modifier, model: SimpleListItemModel, onClick: () -> Unit = {}, ) { - OutlinedCard(modifier = modifier.height(48.dp), onClick = onClick, enabled = model.isEnabled) { - Row(modifier = Modifier.fillMaxSize(), verticalAlignment = Alignment.CenterVertically) { - Spacer(modifier = Modifier.width(16.dp)) + SimpleListItem( + modifier = modifier.height(56.dp), + model = model, + onClick = onClick, + ) +} - when (model.icon) { - is ComposeIconInfo.Vector -> Icon( - modifier = Modifier.size(26.dp), - imageVector = model.icon.imageVector, - contentDescription = null, - tint = LocalContentColor.current, - ) +@Composable +fun SimpleListItem( + modifier: Modifier = Modifier, + model: SimpleListItemModel, + onClick: () -> Unit = {}, +) { + CompositionLocalProvider( + LocalMinimumInteractiveComponentSize provides 16.dp, + ) { + OutlinedCard( + modifier = modifier.height(IntrinsicSize.Min), + onClick = onClick, + enabled = model.isEnabled, + ) { + Row( + modifier = Modifier + .fillMaxSize(), + verticalAlignment = Alignment.CenterVertically, + ) { + Spacer(modifier = Modifier.width(16.dp)) - is ComposeIconInfo.Drawable -> { - val painter = rememberDrawablePainter(model.icon.drawable) - Icon( + when (model.icon) { + is ComposeIconInfo.Vector -> Icon( modifier = Modifier.size(26.dp), - painter = painter, + imageVector = model.icon.imageVector, contentDescription = null, - tint = Color.Unspecified, + tint = LocalContentColor.current, ) - } - } - Spacer(modifier = Modifier.width(16.dp)) + is ComposeIconInfo.Drawable -> { + val painter = rememberDrawablePainter(model.icon.drawable) + Icon( + modifier = Modifier.size(26.dp), + painter = painter, + contentDescription = null, + tint = Color.Unspecified, + ) + } + } - Column( - modifier = Modifier.padding(end = 16.dp), - ) { - Text( - text = model.title, - style = MaterialTheme.typography.bodyMedium, - maxLines = if (model.subtitle == null) { - 2 - } else { - 1 - }, - overflow = TextOverflow.Ellipsis, - ) + Spacer(modifier = Modifier.width(16.dp)) - if (model.subtitle != null) { + Column( + modifier = Modifier + .padding(end = 16.dp) + .heightIn(min = 36.dp), + verticalArrangement = Arrangement.Center, + ) { Text( - text = model.subtitle, - color = if (model.isSubtitleError) MaterialTheme.colorScheme.error else MaterialTheme.colorScheme.onSurface, - style = MaterialTheme.typography.bodySmall, - maxLines = 1, + text = model.title, + style = MaterialTheme.typography.bodyMedium, + maxLines = if (model.subtitle == null) { + 2 + } else { + 1 + }, overflow = TextOverflow.Ellipsis, ) + + if (model.subtitle != null) { + Text( + text = model.subtitle, + color = if (model.isSubtitleError) MaterialTheme.colorScheme.error else MaterialTheme.colorScheme.onSurface, + style = MaterialTheme.typography.bodySmall, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + ) + } } } } diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt new file mode 100644 index 0000000000..05f0f4db8b --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/AdGroup.kt @@ -0,0 +1,78 @@ +package io.github.sds100.keymapper.util.ui.compose.icons + +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.graphics.vector.path +import androidx.compose.ui.unit.dp + +val KeyMapperIcons.AdGroup: ImageVector + get() { + if (_AdGroup != null) { + return _AdGroup!! + } + _AdGroup = ImageVector.Builder( + name = "AdGroup", + defaultWidth = 24.dp, + defaultHeight = 24.dp, + viewportWidth = 960f, + viewportHeight = 960f, + ).apply { + path(fill = SolidColor(Color(0xFF000000))) { + moveTo(320f, 640f) + lineTo(800f, 640f) + quadTo(800f, 640f, 800f, 640f) + quadTo(800f, 640f, 800f, 640f) + lineTo(800f, 240f) + lineTo(320f, 240f) + lineTo(320f, 640f) + quadTo(320f, 640f, 320f, 640f) + quadTo(320f, 640f, 320f, 640f) + close() + moveTo(320f, 720f) + quadTo(287f, 720f, 263.5f, 696.5f) + quadTo(240f, 673f, 240f, 640f) + lineTo(240f, 160f) + quadTo(240f, 127f, 263.5f, 103.5f) + quadTo(287f, 80f, 320f, 80f) + lineTo(800f, 80f) + quadTo(833f, 80f, 856.5f, 103.5f) + quadTo(880f, 127f, 880f, 160f) + lineTo(880f, 640f) + quadTo(880f, 673f, 856.5f, 696.5f) + quadTo(833f, 720f, 800f, 720f) + lineTo(320f, 720f) + close() + moveTo(160f, 880f) + quadTo(127f, 880f, 103.5f, 856.5f) + quadTo(80f, 833f, 80f, 800f) + lineTo(80f, 240f) + lineTo(160f, 240f) + lineTo(160f, 800f) + quadTo(160f, 800f, 160f, 800f) + quadTo(160f, 800f, 160f, 800f) + lineTo(720f, 800f) + lineTo(720f, 880f) + lineTo(160f, 880f) + close() + moveTo(320f, 160f) + quadTo(320f, 160f, 320f, 160f) + quadTo(320f, 160f, 320f, 160f) + lineTo(320f, 640f) + quadTo(320f, 640f, 320f, 640f) + quadTo(320f, 640f, 320f, 640f) + lineTo(320f, 640f) + quadTo(320f, 640f, 320f, 640f) + quadTo(320f, 640f, 320f, 640f) + lineTo(320f, 160f) + quadTo(320f, 160f, 320f, 160f) + quadTo(320f, 160f, 320f, 160f) + close() + } + }.build() + + return _AdGroup!! + } + +@Suppress("ObjectPropertyName") +private var _AdGroup: ImageVector? = null diff --git a/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt new file mode 100644 index 0000000000..498d3bc295 --- /dev/null +++ b/app/src/main/java/io/github/sds100/keymapper/util/ui/compose/icons/JumpToElement.kt @@ -0,0 +1,115 @@ +package io.github.sds100.keymapper.util.ui.compose.icons + +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.graphics.vector.path +import androidx.compose.ui.unit.dp + +val KeyMapperIcons.JumpToElement: ImageVector + get() { + if (_JumpToElement != null) { + return _JumpToElement!! + } + _JumpToElement = ImageVector.Builder( + name = "JumpToElement", + defaultWidth = 24.dp, + defaultHeight = 24.dp, + viewportWidth = 960f, + viewportHeight = 960f, + ).apply { + path(fill = SolidColor(Color(0xFF000000))) { + moveTo(520f, 440f) + lineTo(560f, 440f) + quadTo(577f, 440f, 588.5f, 451.5f) + quadTo(600f, 463f, 600f, 480f) + quadTo(600f, 497f, 588.5f, 508.5f) + quadTo(577f, 520f, 560f, 520f) + lineTo(480f, 520f) + quadTo(463f, 520f, 451.5f, 508.5f) + quadTo(440f, 497f, 440f, 480f) + lineTo(440f, 400f) + quadTo(440f, 383f, 451.5f, 371.5f) + quadTo(463f, 360f, 480f, 360f) + quadTo(497f, 360f, 508.5f, 371.5f) + quadTo(520f, 383f, 520f, 400f) + lineTo(520f, 440f) + close() + moveTo(800f, 440f) + lineTo(800f, 400f) + quadTo(800f, 383f, 811.5f, 371.5f) + quadTo(823f, 360f, 840f, 360f) + quadTo(857f, 360f, 868.5f, 371.5f) + quadTo(880f, 383f, 880f, 400f) + lineTo(880f, 480f) + quadTo(880f, 497f, 868.5f, 508.5f) + quadTo(857f, 520f, 840f, 520f) + lineTo(760f, 520f) + quadTo(743f, 520f, 731.5f, 508.5f) + quadTo(720f, 497f, 720f, 480f) + quadTo(720f, 463f, 731.5f, 451.5f) + quadTo(743f, 440f, 760f, 440f) + lineTo(800f, 440f) + close() + moveTo(520f, 160f) + lineTo(520f, 200f) + quadTo(520f, 217f, 508.5f, 228.5f) + quadTo(497f, 240f, 480f, 240f) + quadTo(463f, 240f, 451.5f, 228.5f) + quadTo(440f, 217f, 440f, 200f) + lineTo(440f, 120f) + quadTo(440f, 103f, 451.5f, 91.5f) + quadTo(463f, 80f, 480f, 80f) + lineTo(560f, 80f) + quadTo(577f, 80f, 588.5f, 91.5f) + quadTo(600f, 103f, 600f, 120f) + quadTo(600f, 137f, 588.5f, 148.5f) + quadTo(577f, 160f, 560f, 160f) + lineTo(520f, 160f) + close() + moveTo(800f, 160f) + lineTo(760f, 160f) + quadTo(743f, 160f, 731.5f, 148.5f) + quadTo(720f, 137f, 720f, 120f) + quadTo(720f, 103f, 731.5f, 91.5f) + quadTo(743f, 80f, 760f, 80f) + lineTo(840f, 80f) + quadTo(857f, 80f, 868.5f, 91.5f) + quadTo(880f, 103f, 880f, 120f) + lineTo(880f, 200f) + quadTo(880f, 217f, 868.5f, 228.5f) + quadTo(857f, 240f, 840f, 240f) + quadTo(823f, 240f, 811.5f, 228.5f) + quadTo(800f, 217f, 800f, 200f) + lineTo(800f, 160f) + close() + moveTo(360f, 656f) + lineTo(164f, 852f) + quadTo(153f, 863f, 136f, 863f) + quadTo(119f, 863f, 108f, 852f) + quadTo(97f, 841f, 97f, 824f) + quadTo(97f, 807f, 108f, 796f) + lineTo(304f, 600f) + lineTo(160f, 600f) + quadTo(143f, 600f, 131.5f, 588.5f) + quadTo(120f, 577f, 120f, 560f) + quadTo(120f, 543f, 131.5f, 531.5f) + quadTo(143f, 520f, 160f, 520f) + lineTo(400f, 520f) + quadTo(417f, 520f, 428.5f, 531.5f) + quadTo(440f, 543f, 440f, 560f) + lineTo(440f, 800f) + quadTo(440f, 817f, 428.5f, 828.5f) + quadTo(417f, 840f, 400f, 840f) + quadTo(383f, 840f, 371.5f, 828.5f) + quadTo(360f, 817f, 360f, 800f) + lineTo(360f, 656f) + close() + } + }.build() + + return _JumpToElement!! + } + +@Suppress("ObjectPropertyName") +private var _JumpToElement: ImageVector? = null diff --git a/app/src/main/res/navigation/nav_app.xml b/app/src/main/res/navigation/nav_app.xml index 4a89c3be6e..61eb7309de 100644 --- a/app/src/main/res/navigation/nav_app.xml +++ b/app/src/main/res/navigation/nav_app.xml @@ -386,4 +386,27 @@ app:argType="string" app:nullable="true" /> + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 46985421d3..933bc823e1 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -24,7 +24,8 @@ Enable accessibility service Restart accessibility service Share - + Nothing here! + Nothing here! Stop repeating when… Trigger is released Trigger is pressed again @@ -98,8 +99,8 @@ Input %s through shell Input %s%s from %s Open %s - Tap coordinates %d, %d - Tap coordinates %d, %d (%s) + Tap screen (%d, %d) + Tap screen (%s) Swipe with %d finger(s) from coordinates %d/%d to %d/%d in %dms Swipe with %d finger(s) from coordinates %d/%d to %d/%d in %dms (%s) %s with %d finger(s) on coordinates %d/%d with a pinch distance of %dpx in %dms @@ -908,6 +909,8 @@ Must be greater than 0! Must be greater than 0! Must be %d or less! + + UI element not found! @@ -1048,6 +1051,7 @@ This action will only work if you have tapped on an input field where the keyboard is supposed to be shown. Show keyboard Hide keyboard + Show keyboard picker Switch keyboard @@ -1061,9 +1065,9 @@ Open settings Show power menu - Toggle Airplane mode - Enable Airplane mode - Disable Airplane mode + Toggle airplane mode + Enable airplane mode + Disable airplane mode Launch app Some devices require apps to have permission before they can launch apps in the background. Tap \"Read more\" to view the instructions on our website. @@ -1098,6 +1102,45 @@ Request body (optional) Authorization header (optional) You must prepend \'Bearer\' if necessary + + Interact with app element + Key Mapper can detect and interact with app elements like menus, tabs, buttons and checkboxes. You need to record yourself interacting with the app element so that Key Mapper knows what you want to do. + Start recording + Stop recording (%s min left) + Go to another app and interact with it. Key Mapper will record what you do and you can choose which interactions you want to use in your key map. Open Key Mapper again when you’re done. + + %d interaction detected + %d interactions detected + + Choose the app to interact with + Record again + Choose app element + Choose the element you want your key map to interact with. + Can\'t find what you’re looking for? + Not all apps are compatible. For incompatible apps you can try the Tap Screen action instead. + Interaction type + Select how you want to interact with the UI element. + Filter interaction type + + Any + Tap + Tap and hold + Focus + Select + Scroll forward + Scroll backward + Expand + Collapse + Unknown: %d + + Interaction details + Description + App + Text / content description + Class name + View resource ID + Unique ID + Interaction types