diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99a81f8ba..2cc42d910 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -373,6 +373,21 @@ jobs: sudo apt-get update -y sudo apt-get install -y ninja-build cmake ccache + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: "17" + distribution: "temurin" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: '8.5' + + - name: Build Kotlin Library + working-directory: source/MaaAndroidControlUnit/java + run: gradle assembleRelease + - uses: nttld/setup-ndk@v1 id: setup-ndk with: @@ -412,6 +427,10 @@ jobs: cp -r LICENSE.md install + # Copy Kotlin AAR + mkdir -p install/android + cp source/MaaAndroidControlUnit/java/build/outputs/aar/*.aar install/android/ || true + - name: Download Plugin uses: robinraju/release-downloader@v1 with: diff --git a/CMakeLists.txt b/CMakeLists.txt index 832cd6097..84d2503a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ include(source/MaaUtils/MaaUtils.cmake) option(WITH_ADB_CONTROLLER "build with adb controller" ON) option(WITH_WIN32_CONTROLLER "build with win32 controller" ON) +option(WITH_ANDROID_CONTROLLER "build with android controller" ON) option(WITH_DBG_CONTROLLER "build with debugging controller" OFF) option(WITH_CUSTOM_CONTROLLER "build with custom controller" ON) option(WITH_NODEJS_BINDING "build with nodejs binding" OFF) @@ -38,6 +39,11 @@ if(WITH_WIN32_CONTROLLER AND NOT WIN32) set(WITH_WIN32_CONTROLLER OFF) endif() +if(WITH_ANDROID_CONTROLLER AND NOT ANDROID) + message(STATUS "Not on Android, disable WITH_ANDROID_CONTROLLER") + set(WITH_ANDROID_CONTROLLER OFF) +endif() + if(WITH_MAA_AGENT) find_package(cppzmq REQUIRED) endif() diff --git a/docs/en_us/2.2-IntegratedInterfaceOverview.md b/docs/en_us/2.2-IntegratedInterfaceOverview.md index 6a1a808a5..5adc3269b 100644 --- a/docs/en_us/2.2-IntegratedInterfaceOverview.md +++ b/docs/en_us/2.2-IntegratedInterfaceOverview.md @@ -221,6 +221,14 @@ Create Adb controller > Screenshot and input methods will be speed tested at startup, selecting the fastest option. +### MaaAndroidControllerCreate + +- `screencap_methods`: bitmask of `MaaAndroidScreencapMethod` (AccessibilityScreenshot or MediaProjection) +- `input_methods`: use `MaaAndroidInputMethod_Accessibility` (control via accessibility service) +- `config`: extra config (reserved, can be empty) + +Create native Android controller (requires accessibility; MediaProjection screenshot needs user consent) + ### MaaWin32ControllerCreate - `hWnd`: window handle diff --git a/docs/en_us/2.4-ControlMethods.md b/docs/en_us/2.4-ControlMethods.md index 8e310fab0..77b4323f3 100644 --- a/docs/en_us/2.4-ControlMethods.md +++ b/docs/en_us/2.4-ControlMethods.md @@ -46,6 +46,27 @@ By default, all methods except `RawByNetcat`, `MinicapDirect`, and `MinicapStrea | MinicapStream | `32` | Takes streaming screenshots and encodes to jpg via minicap tool, transfers via adb process pipe. | | EmulatorExtras | `64` | Uses emulator-specific tools for screenshots. Currently supported emulators: MuMu 12, LDPlayer 9 | +## Android + +### Android Input + +> Reference: `MaaAndroidInputMethod`. + +| Name | Value | Description | +| --- | --- | --- | +| Accessibility | `1` | Uses accessibility service for control, including `dispatchGesture` for tap/swipe/scroll, global actions for key events (back/home/recents), and `ACTION_SET_TEXT` on current focused input node. | + +### Android Screencap + +> Reference: `MaaAndroidScreencapMethod`. + +Combine the selected methods below using **bitwise OR**. Framework will choose available ones, preferring MediaProjection (faster and more stable), then AccessibilityScreenshot. + +| Name | Value | Description | +| --- | --- | --- | +| AccessibilityScreenshot | `1` | API 33+ accessibility `takeScreenshot`. No extra permission required, but needs Android 13 or above. | +| MediaProjection | `2` | MediaProjection virtual display screenshot, requires user consent. Faster and better compatibility. | + ## Win32 ### Win32 Input diff --git "a/docs/zh_cn/2.2-\351\233\206\346\210\220\346\216\245\345\217\243\344\270\200\350\247\210.md" "b/docs/zh_cn/2.2-\351\233\206\346\210\220\346\216\245\345\217\243\344\270\200\350\247\210.md" index 4249f596f..56e2b431c 100644 --- "a/docs/zh_cn/2.2-\351\233\206\346\210\220\346\216\245\345\217\243\344\270\200\350\247\210.md" +++ "b/docs/zh_cn/2.2-\351\233\206\346\210\220\346\216\245\345\217\243\344\270\200\350\247\210.md" @@ -221,6 +221,14 @@ > 截图方式和输入方式会在启动时进行测速，选择最快的方案 +### MaaAndroidControllerCreate + +- `screencap_methods`: 安卓原生截图方式集合，位或自 `MaaAndroidScreencapMethod`（AccessibilityScreenshot 或 MediaProjection） +- `input_methods`: 安卓原生输入方式，使用 `MaaAndroidInputMethod_Accessibility`（通过辅助功能进行控制） +- `config`: 额外配置（预留，可为空字符串） + +创建安卓原生控制器（依赖辅助功能；MediaProjection 截图需用户授权录屏） + ### MaaWin32ControllerCreate - `hWnd`: 窗口句柄 diff --git "a/docs/zh_cn/2.4-\346\216\247\345\210\266\346\226\271\345\274\217\350\257\264\346\230\216.md" "b/docs/zh_cn/2.4-\346\216\247\345\210\266\346\226\271\345\274\217\350\257\264\346\230\216.md" index 99a20dd26..f97f05c6a 100644 --- "a/docs/zh_cn/2.4-\346\216\247\345\210\266\346\226\271\345\274\217\350\257\264\346\230\216.md" +++ "b/docs/zh_cn/2.4-\346\216\247\345\210\266\346\226\271\345\274\217\350\257\264\346\230\216.md" @@ -46,6 +46,27 @@ | MinicapStream | `32` | 通过 minicap 工具流式截图和编码为 jpg，通过 adb 进程管道传输。 | | EmulatorExtras | `64` | 使用模拟器专用工具进行截图。目前支持的模拟器：MuMu 12、雷电 9 | +## Android + +### Android Input + +> 参考 `MaaAndroidInputMethod` 定义。 + +| 名称 | 值 | 说明 | +| --- | --- | --- | +| Accessibility | `1` | 通过辅助功能进行控制，包括 `dispatchGesture` 点击/滑动/滚动、全局按键（返回/主页/多任务）、以及对焦点输入框执行 `ACTION_SET_TEXT`。 | + +### Android Screencap + +> 参考 `MaaAndroidScreencapMethod` 定义。 + +将下面选择的方式 **按位或** 合并为一个值提供。框架会按可用性选择，优先使用 MediaProjection（更快更稳定），其次使用 AccessibilityScreenshot。 + +| 名称 | 值 | 说明 | +| --- | --- | --- | +| AccessibilityScreenshot | `1` | API 33+ 辅助功能 `takeScreenshot`。无需额外权限，但需要 Android 13 及以上。 | +| MediaProjection | `2` | MediaProjection 虚拟显示截图，需要用户授权录屏。速度更快，兼容性更好。 | + ## Win32 ### Win32 Input diff --git a/include/MaaFramework/Instance/MaaController.h b/include/MaaFramework/Instance/MaaController.h index 5c51133f4..53c92a974 100644 --- a/include/MaaFramework/Instance/MaaController.h +++ b/include/MaaFramework/Instance/MaaController.h @@ -29,6 +29,9 @@ extern "C" const char* config, const char* agent_path); + MAA_FRAMEWORK_API MaaController* + MaaAndroidControllerCreate(MaaAndroidScreencapMethod screencap_methods, MaaAndroidInputMethod input_methods); + MAA_FRAMEWORK_API MaaController* MaaWin32ControllerCreate( void* hWnd, MaaWin32ScreencapMethod screencap_method, diff --git a/include/MaaFramework/MaaDef.h b/include/MaaFramework/MaaDef.h index 7b34d79ef..e155c7a77 100644 --- a/include/MaaFramework/MaaDef.h +++ b/include/MaaFramework/MaaDef.h @@ -249,6 +249,27 @@ typedef uint64_t MaaAdbInputMethod; #define MaaAdbInputMethod_All (~MaaAdbInputMethod_None) #define MaaAdbInputMethod_Default (MaaAdbInputMethod_All & (~MaaAdbInputMethod_EmulatorExtras)) +// MaaAndroidScreencapMethod: +/** + * Use bitwise OR to set the method you need, MaaFramework will test their availability. + */ +typedef uint64_t MaaAndroidScreencapMethod; +#define MaaAndroidScreencapMethod_None 0ULL +/// API 33+ AccessibilityService.takeScreenshot +#define MaaAndroidScreencapMethod_AccessibilityScreenshot 1ULL +/// MediaProjection VirtualDisplay capture (requires user consent) +#define MaaAndroidScreencapMethod_MediaProjection (1ULL << 1) +#define MaaAndroidScreencapMethod_All (~MaaAndroidScreencapMethod_None) +#define MaaAndroidScreencapMethod_Default (MaaAndroidScreencapMethod_AccessibilityScreenshot | MaaAndroidScreencapMethod_MediaProjection) + +// MaaAndroidInputMethod: +/** + * Use bitwise OR to set the method you need. + */ +typedef uint64_t MaaAndroidInputMethod; +#define MaaAndroidInputMethod_None 0ULL +#define MaaAndroidInputMethod_Accessibility 1ULL + // MaaWin32ScreencapMethod: /** * No bitwise OR, just set it diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 3b1f60279..e056d4f89 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -2,6 +2,10 @@ if(WITH_ADB_CONTROLLER) add_subdirectory(MaaAdbControlUnit) endif() +if(WITH_ANDROID_CONTROLLER) + add_subdirectory(MaaAndroidControlUnit) +endif() + if(WITH_WIN32_CONTROLLER) add_subdirectory(MaaWin32ControlUnit) endif() diff --git a/source/LibraryHolder/CMakeLists.txt b/source/LibraryHolder/CMakeLists.txt index 98bf6a05e..7797ed84c 100644 --- a/source/LibraryHolder/CMakeLists.txt +++ b/source/LibraryHolder/CMakeLists.txt @@ -15,6 +15,10 @@ if(WITH_ADB_CONTROLLER) add_dependencies(LibraryHolder MaaAdbControlUnit) endif() +if(WITH_ANDROID_CONTROLLER) + add_dependencies(LibraryHolder MaaAndroidControlUnit) +endif() + if(WITH_WIN32_CONTROLLER) add_dependencies(LibraryHolder MaaWin32ControlUnit) endif() diff --git a/source/LibraryHolder/ControlUnit/ControlUnit.cpp b/source/LibraryHolder/ControlUnit/ControlUnit.cpp index 16a1e2fcf..ceef788fa 100644 --- a/source/LibraryHolder/ControlUnit/ControlUnit.cpp +++ b/source/LibraryHolder/ControlUnit/ControlUnit.cpp @@ -3,6 +3,7 @@ #include #include "ControlUnit/AdbControlUnitAPI.h" +#include "ControlUnit/AndroidControlUnitAPI.h" #include "ControlUnit/CustomControlUnitAPI.h" #include "ControlUnit/DbgControlUnitAPI.h" #include "ControlUnit/Win32ControlUnitAPI.h" @@ -65,6 +66,39 @@ std::shared_ptr AdbControlUnitLibraryHolder return std::shared_ptr(control_unit_handle, destroy_control_unit_func); } +std::shared_ptr AndroidControlUnitLibraryHolder::create_control_unit( + MaaAndroidScreencapMethod screencap_methods, + MaaAndroidInputMethod input_methods) +{ + if (!load_library(library_dir() / libname_)) { + LogError << "Failed to load library" << VAR(library_dir()) << VAR(libname_); + return nullptr; + } + + check_version(version_func_name_); + + auto create_control_unit_func = get_function(create_func_name_); + if (!create_control_unit_func) { + LogError << "Failed to get function create_control_unit"; + return nullptr; + } + + auto destroy_control_unit_func = get_function(destroy_func_name_); + if (!destroy_control_unit_func) { + LogError << "Failed to get function destroy_control_unit"; + return nullptr; + } + + auto control_unit_handle = create_control_unit_func(screencap_methods, input_methods); + + if (!control_unit_handle) { + LogError << "Failed to create control unit"; + return nullptr; + } + + return std::shared_ptr(control_unit_handle, destroy_control_unit_func); +} + std::shared_ptr Win32ControlUnitLibraryHolder::create_control_unit( void* hWnd, MaaWin32ScreencapMethod screencap_method, diff --git a/source/MaaAndroidControlUnit/API/AndroidControlUnitAPI.cpp b/source/MaaAndroidControlUnit/API/AndroidControlUnitAPI.cpp new file mode 100644 index 000000000..d7264d57b --- /dev/null +++ b/source/MaaAndroidControlUnit/API/AndroidControlUnitAPI.cpp @@ -0,0 +1,35 @@ +#include "ControlUnit/AndroidControlUnitAPI.h" + +#include + +#include "MaaUtils/Logger.h" +#include "Manager/AndroidControlUnitMgr.h" + +const char* MaaAndroidControlUnitGetVersion() +{ +#pragma message("MaaAndroidControlUnit MAA_VERSION: " MAA_VERSION) + + return MAA_VERSION; +} + +MaaAndroidControlUnitHandle MaaAndroidControlUnitCreate( + MaaAndroidScreencapMethod screencap_methods, + MaaAndroidInputMethod input_methods) +{ + using namespace MAA_CTRL_UNIT_NS; + + LogFunc << VAR(screencap_methods) << VAR(input_methods); + + auto unit_mgr = std::make_unique(screencap_methods, input_methods); + + return unit_mgr.release(); +} + +void MaaAndroidControlUnitDestroy(MaaAndroidControlUnitHandle handle) +{ + LogFunc << VAR_VOIDP(handle); + + if (handle) { + delete handle; + } +} diff --git a/source/MaaAndroidControlUnit/CMakeLists.txt b/source/MaaAndroidControlUnit/CMakeLists.txt new file mode 100644 index 000000000..ce01f782d --- /dev/null +++ b/source/MaaAndroidControlUnit/CMakeLists.txt @@ -0,0 +1,28 @@ +if(NOT ANDROID) + message(STATUS "MaaAndroidControlUnit skipped: not on Android") + return() +endif() + +file(GLOB_RECURSE maa_android_control_unit_src *.h *.hpp *.cpp) +file(GLOB_RECURSE maa_android_control_unit_header ${MAA_PRIVATE_INC}/ControlUnit/AndroidControlUnitAPI.h ${MAA_PRIVATE_INC}/ControlUnit/ControlUnitAPI.h) + +add_library(MaaAndroidControlUnit SHARED ${maa_android_control_unit_src} ${maa_android_control_unit_header}) + +target_include_directories(MaaAndroidControlUnit + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${MAA_PRIVATE_INC} ${MAA_PUBLIC_INC}) + +target_compile_definitions(MaaAndroidControlUnit PRIVATE MAA_CONTROL_UNIT_EXPORTS) + +target_link_libraries(MaaAndroidControlUnit + PRIVATE MaaUtils HeaderOnlyLibraries ${OpenCV_LIBS} log jnigraphics android) + +add_dependencies(MaaAndroidControlUnit MaaUtils) + +install( + TARGETS MaaAndroidControlUnit + RUNTIME DESTINATION bin + LIBRARY DESTINATION bin + # ARCHIVE DESTINATION lib +) + +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${maa_android_control_unit_src}) diff --git a/source/MaaAndroidControlUnit/Manager/AndroidControlUnitMgr.cpp b/source/MaaAndroidControlUnit/Manager/AndroidControlUnitMgr.cpp new file mode 100644 index 000000000..d0ddec329 --- /dev/null +++ b/source/MaaAndroidControlUnit/Manager/AndroidControlUnitMgr.cpp @@ -0,0 +1,349 @@ +#include "AndroidControlUnitMgr.h" + +#include +#include + +#include "MaaUtils/Logger.h" + +MAA_CTRL_UNIT_NS_BEGIN + +namespace +{ +JavaVM* g_vm = nullptr; +} + +AndroidControlUnitMgr::AndroidControlUnitMgr(MaaAndroidScreencapMethod screencap_methods, MaaAndroidInputMethod input_methods) + : screencap_methods_(screencap_methods) + , input_methods_(input_methods) + , vm_(g_vm) +{ +} + +JNIEnv* AndroidControlUnitMgr::ensure_env() +{ + if (!vm_) { + vm_ = g_vm; + if (!vm_) { + return nullptr; + } + } + JNIEnv* env = nullptr; + if (vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) == JNI_OK) { + return env; + } + if (vm_->AttachCurrentThread(&env, nullptr) != JNI_OK) { + return nullptr; + } + return env; +} + +jclass AndroidControlUnitMgr::bridge_class() +{ + if (bridge_cls_) { + return bridge_cls_; + } + JNIEnv* env = ensure_env(); + if (!env) { + return nullptr; + } + jclass local = env->FindClass("com/maa/framework/nativectrl/NativeBridge"); + if (env->ExceptionCheck() || !local) { + env->ExceptionClear(); + LogError << "NativeBridge class not found"; + return nullptr; + } + bridge_cls_ = reinterpret_cast(env->NewGlobalRef(local)); + env->DeleteLocalRef(local); + return bridge_cls_; +} + +template +bool AndroidControlUnitMgr::call_bool(const char* name, const char* sig, Callable&& caller) +{ + JNIEnv* env = ensure_env(); + if (!env) { + LogError << "JNIEnv null"; + return false; + } + jclass cls = bridge_class(); + if (!cls) { + return false; + } + jmethodID mid = env->GetStaticMethodID(cls, name, sig); + if (!mid) { + LogError << "Method not found" << name; + env->ExceptionClear(); + return false; + } + jboolean ok = caller(env, cls, mid); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + return false; + } + return ok == JNI_TRUE; +} + +bool AndroidControlUnitMgr::connect() +{ + // First call init to pass screencap_methods and input_methods to Kotlin side + JNIEnv* env = ensure_env(); + if (!env) { + LogError << "JNIEnv null"; + return false; + } + jclass cls = bridge_class(); + if (!cls) { + return false; + } + + // Call init(screencapMethods, inputMethods) + jmethodID init_mid = env->GetStaticMethodID(cls, "init", "(JJ)Z"); + if (!init_mid) { + LogError << "Method init not found"; + env->ExceptionClear(); + return false; + } + jboolean init_ok = + env->CallStaticBooleanMethod(cls, init_mid, static_cast(screencap_methods_), static_cast(input_methods_)); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + return false; + } + if (!init_ok) { + LogError << "init failed"; + return false; + } + + // Call connect() + return call_bool("connect", "()Z", [](JNIEnv* env, jclass cls, jmethodID mid) { return env->CallStaticBooleanMethod(cls, mid); }); +} + +bool AndroidControlUnitMgr::request_uuid(/*out*/ std::string& uuid) +{ + JNIEnv* env = ensure_env(); + if (!env) { + LogError << "JNIEnv null"; + return false; + } + jclass cls = bridge_class(); + if (!cls) { + return false; + } + jmethodID mid = env->GetStaticMethodID(cls, "requestUuid", "()Ljava/lang/String;"); + if (!mid) { + LogError << "Method requestUuid not found"; + env->ExceptionClear(); + return false; + } + auto str = static_cast(env->CallStaticObjectMethod(cls, mid)); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + return false; + } + if (!str) { + return false; + } + const char* chars = env->GetStringUTFChars(str, nullptr); + uuid = chars ? chars : ""; + if (chars) { + env->ReleaseStringUTFChars(str, chars); + } + env->DeleteLocalRef(str); + return !uuid.empty(); +} + +MaaControllerFeature AndroidControlUnitMgr::get_features() const +{ + return MaaControllerFeature_None; +} + +bool AndroidControlUnitMgr::start_app(const std::string& intent) +{ + return call_bool("startApp", "(Ljava/lang/String;)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + jstring jintent = env->NewStringUTF(intent.c_str()); + jboolean ok = env->CallStaticBooleanMethod(cls, mid, jintent); + env->DeleteLocalRef(jintent); + return ok; + }); +} + +bool AndroidControlUnitMgr::stop_app(const std::string& intent) +{ + return call_bool("stopApp", "(Ljava/lang/String;)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + jstring jintent = env->NewStringUTF(intent.c_str()); + jboolean ok = env->CallStaticBooleanMethod(cls, mid, jintent); + env->DeleteLocalRef(jintent); + return ok; + }); +} + +bool AndroidControlUnitMgr::screencap(/*out*/ cv::Mat& image) +{ + JNIEnv* env = ensure_env(); + if (!env) { + LogError << "JNIEnv null"; + return false; + } + jclass cls = bridge_class(); + if (!cls) { + return false; + } + jmethodID mid = env->GetStaticMethodID(cls, "screencap", "()Landroid/graphics/Bitmap;"); + if (!mid) { + LogError << "Method screencap not found"; + env->ExceptionClear(); + return false; + } + jobject bmp = env->CallStaticObjectMethod(cls, mid); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + return false; + } + if (!bmp) { + return false; + } + + // Helper lambda to recycle bitmap and delete local ref + auto cleanup_bitmap = [env](jobject bitmap) { + // Call Bitmap.recycle() to release native memory + jclass bitmap_class = env->GetObjectClass(bitmap); + if (bitmap_class) { + jmethodID recycle_mid = env->GetMethodID(bitmap_class, "recycle", "()V"); + if (recycle_mid) { + env->CallVoidMethod(bitmap, recycle_mid); + } + env->DeleteLocalRef(bitmap_class); + } + env->DeleteLocalRef(bitmap); + }; + + AndroidBitmapInfo info {}; + if (AndroidBitmap_getInfo(env, bmp, &info) != ANDROID_BITMAP_RESULT_SUCCESS) { + cleanup_bitmap(bmp); + return false; + } + + if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888 && info.format != ANDROID_BITMAP_FORMAT_RGB_565) { + LogError << "Unsupported bitmap format" << info.format; + cleanup_bitmap(bmp); + return false; + } + + void* pixels = nullptr; + if (AndroidBitmap_lockPixels(env, bmp, &pixels) != ANDROID_BITMAP_RESULT_SUCCESS || !pixels) { + cleanup_bitmap(bmp); + return false; + } + + const int cv_type = CV_8UC4; + image.create(static_cast(info.height), static_cast(info.width), cv_type); + + if (info.format == ANDROID_BITMAP_FORMAT_RGBA_8888) { + const size_t row_bytes = info.stride; + for (uint32_t row = 0; row < info.height; ++row) { + std::memcpy(image.ptr(row), static_cast(pixels) + row * row_bytes, image.cols * 4); + } + } + else { + // RGB565 -> RGBA8888 expansion + for (uint32_t row = 0; row < info.height; ++row) { + const uint16_t* src = reinterpret_cast(static_cast(pixels) + row * info.stride); + uint8_t* dst = image.ptr(row); + for (uint32_t col = 0; col < info.width; ++col) { + uint16_t v = src[col]; + uint8_t r = ((v >> 11) & 0x1F) << 3; + uint8_t g = ((v >> 5) & 0x3F) << 2; + uint8_t b = (v & 0x1F) << 3; + dst[col * 4 + 0] = b; + dst[col * 4 + 1] = g; + dst[col * 4 + 2] = r; + dst[col * 4 + 3] = 255; + } + } + } + + AndroidBitmap_unlockPixels(env, bmp); + cleanup_bitmap(bmp); + return true; +} + +bool AndroidControlUnitMgr::click(int x, int y) +{ + return call_bool("tap", "(II)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { return env->CallStaticBooleanMethod(cls, mid, x, y); }); +} + +bool AndroidControlUnitMgr::swipe(int x1, int y1, int x2, int y2, int duration) +{ + return call_bool("swipe", "(IIIII)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + return env->CallStaticBooleanMethod(cls, mid, x1, y1, x2, y2, duration); + }); +} + +bool AndroidControlUnitMgr::touch_down(int contact, int x, int y, int pressure) +{ + std::ignore = contact; + std::ignore = pressure; + return click(x, y); +} + +bool AndroidControlUnitMgr::touch_move(int contact, int x, int y, int pressure) +{ + std::ignore = contact; + std::ignore = pressure; + std::ignore = x; + std::ignore = y; + return true; +} + +bool AndroidControlUnitMgr::touch_up(int contact) +{ + std::ignore = contact; + return true; +} + +bool AndroidControlUnitMgr::click_key(int key) +{ + return call_bool("keyEvent", "(IZ)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + return env->CallStaticBooleanMethod(cls, mid, key, static_cast(true)); + }); +} + +bool AndroidControlUnitMgr::input_text(const std::string& text) +{ + return call_bool("inputText", "(Ljava/lang/String;)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + jstring jtext = env->NewStringUTF(text.c_str()); + jboolean ok = env->CallStaticBooleanMethod(cls, mid, jtext); + env->DeleteLocalRef(jtext); + return ok; + }); +} + +bool AndroidControlUnitMgr::key_down(int key) +{ + return call_bool("keyEvent", "(IZ)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + return env->CallStaticBooleanMethod(cls, mid, key, static_cast(true)); + }); +} + +bool AndroidControlUnitMgr::key_up(int key) +{ + return call_bool("keyEvent", "(IZ)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + return env->CallStaticBooleanMethod(cls, mid, key, static_cast(false)); + }); +} + +bool AndroidControlUnitMgr::scroll(int dx, int dy) +{ + return call_bool("scroll", "(II)Z", [&](JNIEnv* env, jclass cls, jmethodID mid) { + return env->CallStaticBooleanMethod(cls, mid, dx, dy); + }); +} + +MAA_CTRL_UNIT_NS_END + +// JNI_OnLoad to capture JavaVM +JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) +{ + MAA_CTRL_UNIT_NS::g_vm = vm; + return JNI_VERSION_1_6; +} diff --git a/source/MaaAndroidControlUnit/Manager/AndroidControlUnitMgr.h b/source/MaaAndroidControlUnit/Manager/AndroidControlUnitMgr.h new file mode 100644 index 000000000..68b26e9c3 --- /dev/null +++ b/source/MaaAndroidControlUnit/Manager/AndroidControlUnitMgr.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "ControlUnit/AndroidControlUnitAPI.h" + +#include "Common/Conf.h" + +MAA_CTRL_UNIT_NS_BEGIN + +class AndroidControlUnitMgr + : public AndroidControlUnitAPI +{ +public: + AndroidControlUnitMgr(MaaAndroidScreencapMethod screencap_methods, MaaAndroidInputMethod input_methods); + virtual ~AndroidControlUnitMgr() override = default; + +public: // from ControlUnitAPI + virtual bool connect() override; + + virtual bool request_uuid(/*out*/ std::string& uuid) override; + virtual MaaControllerFeature get_features() const override; + + virtual bool start_app(const std::string& intent) override; + virtual bool stop_app(const std::string& intent) override; + + virtual bool screencap(/*out*/ cv::Mat& image) override; + + virtual bool click(int x, int y) override; + virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + + virtual bool touch_down(int contact, int x, int y, int pressure) override; + virtual bool touch_move(int contact, int x, int y, int pressure) override; + virtual bool touch_up(int contact) override; + + virtual bool click_key(int key) override; + virtual bool input_text(const std::string& text) override; + + virtual bool key_down(int key) override; + virtual bool key_up(int key) override; + + virtual bool scroll(int dx, int dy) override; + +private: + // JNI helpers + JNIEnv* ensure_env(); + jclass bridge_class(); + + template + bool call_bool(const char* name, const char* sig, Callable&& caller); + +private: + const MaaAndroidScreencapMethod screencap_methods_ = MaaAndroidScreencapMethod_None; + const MaaAndroidInputMethod input_methods_ = MaaAndroidInputMethod_None; + + JavaVM* vm_ = nullptr; + jclass bridge_cls_ = nullptr; +}; + +MAA_CTRL_UNIT_NS_END diff --git a/source/MaaAndroidControlUnit/java/AndroidManifest.xml b/source/MaaAndroidControlUnit/java/AndroidManifest.xml new file mode 100644 index 000000000..db47ac240 --- /dev/null +++ b/source/MaaAndroidControlUnit/java/AndroidManifest.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + diff --git a/source/MaaAndroidControlUnit/java/build.gradle.kts b/source/MaaAndroidControlUnit/java/build.gradle.kts new file mode 100644 index 000000000..08e7c7cfd --- /dev/null +++ b/source/MaaAndroidControlUnit/java/build.gradle.kts @@ -0,0 +1,44 @@ +plugins { + id("com.android.library") + id("org.jetbrains.kotlin.android") +} + +android { + namespace = "com.maa.framework.nativectrl" + compileSdk = 34 + + defaultConfig { + minSdk = 24 + + consumerProguardFiles("consumer-rules.pro") + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + kotlinOptions { + jvmTarget = "17" + } + + sourceSets { + getByName("main") { + java.srcDirs("com") + kotlin.srcDirs("com") + manifest.srcFile("AndroidManifest.xml") + res.srcDirs("res") + } + } +} + +dependencies { + implementation("androidx.core:core-ktx:1.12.0") +} diff --git a/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MaaAccessibilityService.kt b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MaaAccessibilityService.kt new file mode 100644 index 000000000..4f48762d2 --- /dev/null +++ b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MaaAccessibilityService.kt @@ -0,0 +1,202 @@ +package com.maa.framework.nativectrl + +import android.accessibilityservice.AccessibilityService +import android.accessibilityservice.GestureDescription +import android.accessibilityservice.GestureDescription.StrokeDescription +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Path +import android.os.Build +import android.os.Bundle +import android.util.DisplayMetrics +import android.view.Display +import android.view.WindowManager +import android.view.accessibility.AccessibilityEvent +import android.view.accessibility.AccessibilityNodeInfo +import android.view.KeyEvent +import java.util.UUID +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit + +class MaaAccessibilityService : AccessibilityService() { + + private val uuidCache: String by lazy { loadUuid() } + + private var cachedScreenWidth = 0 + private var cachedScreenHeight = 0 + + private val windowMgr: WindowManager by lazy { + getSystemService(Context.WINDOW_SERVICE) as WindowManager + } + + override fun onServiceConnected() { + super.onServiceConnected() + updateScreenSize() + NativeBridge.attachAccessibilityService(this) + } + + override fun onUnbind(intent: android.content.Intent?): Boolean { + NativeBridge.detachAccessibilityService(this) + return super.onUnbind(intent) + } + + override fun onAccessibilityEvent(event: AccessibilityEvent?) { + // No-op + } + + override fun onInterrupt() { + // No-op + } + + private fun updateScreenSize() { + val metrics = DisplayMetrics() + @Suppress("DEPRECATION") + windowMgr.defaultDisplay.getRealMetrics(metrics) + cachedScreenWidth = metrics.widthPixels + cachedScreenHeight = metrics.heightPixels + } + + fun getScreenSize(): IntArray { + if (cachedScreenWidth == 0 || cachedScreenHeight == 0) { + updateScreenSize() + } + return intArrayOf(cachedScreenWidth, cachedScreenHeight) + } + + fun uuid(): String = uuidCache + + fun startApp(intentUri: String): Boolean { + return try { + val intent = android.content.Intent.parseUri(intentUri, 0).apply { + addFlags(android.content.Intent.FLAG_ACTIVITY_NEW_TASK) + } + startActivity(intent) + true + } catch (_: Exception) { + false + } + } + + @Suppress("UNUSED_PARAMETER") + fun stopApp(intentUri: String): Boolean { + // Best effort: bring home + performGlobalAction(GLOBAL_ACTION_HOME) + return true + } + + fun tap(x: Int, y: Int): Boolean = dispatchGestureOnce( + gesturePath { moveTo(x.toFloat(), y.toFloat()); lineTo(x.toFloat(), y.toFloat()) }, + 100L + ) + + fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, duration: Int): Boolean = + dispatchGestureOnce( + gesturePath(duration.toLong().coerceAtLeast(50L)) { + moveTo(x1.toFloat(), y1.toFloat()) + lineTo(x2.toFloat(), y2.toFloat()) + }, + duration.toLong().coerceAtLeast(50L) + ) + + fun scroll(dx: Int, dy: Int): Boolean { + val metrics = DisplayMetrics() + @Suppress("DEPRECATION") + windowMgr.defaultDisplay.getRealMetrics(metrics) + val startX = (metrics.widthPixels / 2f + dx * 0.25f) + val startY = (metrics.heightPixels / 2f + dy * 0.25f) + val endX = startX - dx + val endY = startY - dy + return dispatchGestureOnce( + gesturePath(150L) { moveTo(startX, startY); lineTo(endX, endY) }, + 150L + ) + } + + fun keyEvent(key: Int, down: Boolean): Boolean { + // Only trigger on key down to avoid double actions + if (!down) return true + return when (key) { + KeyEvent.KEYCODE_BACK -> performGlobalAction(GLOBAL_ACTION_BACK) + KeyEvent.KEYCODE_HOME -> performGlobalAction(GLOBAL_ACTION_HOME) + KeyEvent.KEYCODE_APP_SWITCH -> performGlobalAction(GLOBAL_ACTION_RECENTS) + else -> false + } + } + + fun inputText(text: String): Boolean { + val root = rootInActiveWindow ?: return false + val node = root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT) + if (node == null) { + root.recycle() + return false + } + val args = Bundle() + args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text) + val result = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args) + node.recycle() + root.recycle() + return result + } + + fun screencap(): Bitmap? { + if (Build.VERSION.SDK_INT < 33) return null + val future = CompletableFuture() + takeScreenshot(Display.DEFAULT_DISPLAY, mainExecutor, + object : TakeScreenshotCallback { + override fun onSuccess(result: ScreenshotResult) { + val hw = result.hardwareBuffer + val bmp = Bitmap.wrapHardwareBuffer(hw, result.colorSpace) + val copy = bmp?.copy(Bitmap.Config.ARGB_8888, false) + bmp?.recycle() + hw.close() + future.complete(copy) + } + override fun onFailure(errorCode: Int) { + future.complete(null) + } + } + ) + return try { + future.get(1200, TimeUnit.MILLISECONDS) + } catch (_: Exception) { + null + } + } + + private fun gesturePath(durationMs: Long = 100L, builder: Path.() -> Unit): GestureDescription { + val path = Path().apply(builder) + val stroke = StrokeDescription(path, 0, durationMs) + return GestureDescription.Builder().addStroke(stroke).build() + } + + private fun dispatchGestureOnce(gesture: GestureDescription, durationMs: Long): Boolean { + val latch = CompletableFuture() + dispatchGesture( + gesture, + object : GestureResultCallback() { + override fun onCompleted(gestureDescription: GestureDescription) { + latch.complete(true) + } + + override fun onCancelled(gestureDescription: GestureDescription) { + latch.complete(false) + } + }, + null + ) + return try { + latch.get(durationMs + 500, TimeUnit.MILLISECONDS) + } catch (_: Exception) { + false + } + } + + private fun loadUuid(): String { + val prefs = getSharedPreferences("maa_ctrl", MODE_PRIVATE) + val cached = prefs.getString("uuid", null) + if (cached != null) return cached + val generated = UUID.randomUUID().toString() + prefs.edit().putString("uuid", generated).apply() + return generated + } +} diff --git a/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MediaProjectionActivity.kt b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MediaProjectionActivity.kt new file mode 100644 index 000000000..e47d27dff --- /dev/null +++ b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MediaProjectionActivity.kt @@ -0,0 +1,82 @@ +package com.maa.framework.nativectrl + +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.media.projection.MediaProjectionManager +import android.os.Bundle + +/** + * Transparent activity to request MediaProjection permission. + * This is launched from native code when MediaProjection is needed. + */ +class MediaProjectionActivity : Activity() { + + companion object { + private const val REQUEST_CODE_CAPTURE = 1001 + + @Volatile + private var pendingCallback: ((Boolean) -> Unit)? = null + + fun requestPermission(context: Context, callback: (Boolean) -> Unit) { + pendingCallback = callback + val intent = Intent(context, MediaProjectionActivity::class.java).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + context.startActivity(intent) + } + + // 清理静态回调，防止内存泄漏 + internal fun clearCallback() { + pendingCallback = null + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + // 检查是否有待处理的回调，如果没有则直接关闭 + if (pendingCallback == null) { + finish() + return + } + + val projectionManager = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + @Suppress("DEPRECATION") + startActivityForResult(projectionManager.createScreenCaptureIntent(), REQUEST_CODE_CAPTURE) + } + + @Deprecated("Deprecated in Java") + override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { + super.onActivityResult(requestCode, resultCode, data) + + if (requestCode == REQUEST_CODE_CAPTURE) { + val callback = pendingCallback + pendingCallback = null // 立即清理回调引用 + + val success = if (resultCode == RESULT_OK && data != null) { + val projectionManager = getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + val projection = projectionManager.getMediaProjection(resultCode, data) + if (projection != null) { + val holder = MediaProjectionHolder(applicationContext, projection) + NativeBridge.attachMediaProjection(holder) + true + } else { + false + } + } else { + false + } + + callback?.invoke(success) + } + + finish() + } + + override fun onDestroy() { + super.onDestroy() + // 确保在 Activity 销毁时清理回调 + pendingCallback = null + } +} diff --git a/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MediaProjectionHolder.kt b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MediaProjectionHolder.kt new file mode 100644 index 000000000..931dff95a --- /dev/null +++ b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/MediaProjectionHolder.kt @@ -0,0 +1,152 @@ +package com.maa.framework.nativectrl + +import android.annotation.SuppressLint +import android.content.Context +import android.graphics.Bitmap +import android.graphics.PixelFormat +import android.hardware.display.DisplayManager +import android.hardware.display.VirtualDisplay +import android.media.Image +import android.media.ImageReader +import android.media.projection.MediaProjection +import android.os.Handler +import android.os.HandlerThread +import android.util.DisplayMetrics +import android.view.WindowManager +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +class MediaProjectionHolder( + private val context: Context, + private val mediaProjection: MediaProjection +) { + private var virtualDisplay: VirtualDisplay? = null + private var imageReader: ImageReader? = null + private var handlerThread: HandlerThread? = null + private var handler: Handler? = null + + private var screenWidth = 0 + private var screenHeight = 0 + private var screenDensity = 0 + + init { + initScreen() + setupImageReader() + } + + @SuppressLint("WrongConstant") + private fun initScreen() { + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + val metrics = DisplayMetrics() + @Suppress("DEPRECATION") + wm.defaultDisplay.getRealMetrics(metrics) + screenWidth = metrics.widthPixels + screenHeight = metrics.heightPixels + screenDensity = metrics.densityDpi + } + + @SuppressLint("WrongConstant") + private fun setupImageReader() { + handlerThread = HandlerThread("MediaProjectionHandler").apply { start() } + handler = Handler(handlerThread!!.looper) + + imageReader = ImageReader.newInstance( + screenWidth, + screenHeight, + PixelFormat.RGBA_8888, + 2 + ) + + virtualDisplay = mediaProjection.createVirtualDisplay( + "MaaScreenCapture", + screenWidth, + screenHeight, + screenDensity, + DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, + imageReader!!.surface, + null, + handler + ) + } + + fun capture(): Bitmap? { + val reader = imageReader ?: return null + + // 首先尝试获取已有的图像 + var image: Image? = try { + reader.acquireLatestImage() + } catch (_: Exception) { + null + } + + // 如果没有现成的图像，等待下一帧 + if (image == null) { + val imageRef = AtomicReference() + val latch = CountDownLatch(1) + + reader.setOnImageAvailableListener({ ir -> + try { + imageRef.set(ir.acquireLatestImage()) + } catch (_: Exception) { + } finally { + latch.countDown() + } + }, handler) + + if (!latch.await(500, TimeUnit.MILLISECONDS)) { + reader.setOnImageAvailableListener(null, null) + return null + } + reader.setOnImageAvailableListener(null, null) + + image = imageRef.get() + } + + if (image == null) return null + + return try { + imageToBitmap(image) + } finally { + image.close() + } + } + + private fun imageToBitmap(image: Image): Bitmap? { + val planes = image.planes + val buffer = planes[0].buffer + val pixelStride = planes[0].pixelStride + val rowStride = planes[0].rowStride + val rowPadding = rowStride - pixelStride * image.width + + val bitmap = Bitmap.createBitmap( + image.width + rowPadding / pixelStride, + image.height, + Bitmap.Config.ARGB_8888 + ) + bitmap.copyPixelsFromBuffer(buffer) + + // Crop if needed + return if (rowPadding > 0) { + val cropped = Bitmap.createBitmap(bitmap, 0, 0, image.width, image.height) + bitmap.recycle() // 回收原始 bitmap 避免内存泄漏 + cropped + } else { + bitmap + } + } + + fun release() { + virtualDisplay?.release() + virtualDisplay = null + + imageReader?.close() + imageReader = null + + handlerThread?.quitSafely() + handlerThread = null + handler = null + + mediaProjection.stop() + } +} diff --git a/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/NativeBridge.kt b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/NativeBridge.kt new file mode 100644 index 000000000..f44429dbc --- /dev/null +++ b/source/MaaAndroidControlUnit/java/com/maa/framework/nativectrl/NativeBridge.kt @@ -0,0 +1,130 @@ +package com.maa.framework.nativectrl + +import android.graphics.Bitmap + +object NativeBridge { + + // Screencap methods - matches MaaDef.h + const val SCREENCAP_METHOD_NONE = 0L + const val SCREENCAP_METHOD_ACCESSIBILITY = 1L + const val SCREENCAP_METHOD_MEDIA_PROJECTION = 1L shl 1 + + // Input methods - matches MaaDef.h + const val INPUT_METHOD_NONE = 0L + const val INPUT_METHOD_ACCESSIBILITY = 1L + + @Volatile + private var accessibilityService: MaaAccessibilityService? = null + + @Volatile + private var mediaProjectionHolder: MediaProjectionHolder? = null + + @Volatile + private var screencapMethods: Long = SCREENCAP_METHOD_NONE + + @Volatile + private var inputMethods: Long = INPUT_METHOD_NONE + + @JvmStatic + fun attachAccessibilityService(service: MaaAccessibilityService) { + this.accessibilityService = service + } + + @JvmStatic + fun detachAccessibilityService(service: MaaAccessibilityService) { + if (this.accessibilityService == service) { + this.accessibilityService = null + } + } + + @JvmStatic + fun attachMediaProjection(holder: MediaProjectionHolder) { + this.mediaProjectionHolder = holder + } + + @JvmStatic + fun detachMediaProjection() { + mediaProjectionHolder?.release() + mediaProjectionHolder = null + } + + // Called from JNI to initialize with methods + @JvmStatic + fun init(screencapMethods: Long, inputMethods: Long): Boolean { + this.screencapMethods = screencapMethods + this.inputMethods = inputMethods + return true + } + + @JvmStatic + fun connect(): Boolean { + // Check if accessibility service is available when needed + val needAccessibility = (screencapMethods and SCREENCAP_METHOD_ACCESSIBILITY) != 0L || + (inputMethods and INPUT_METHOD_ACCESSIBILITY) != 0L + if (needAccessibility && accessibilityService == null) { + return false + } + + // MediaProjection will be requested on demand + return true + } + + @JvmStatic + fun requestUuid(): String = accessibilityService?.uuid().orEmpty() + + @JvmStatic + fun startApp(intent: String): Boolean = accessibilityService?.startApp(intent) ?: false + + @JvmStatic + fun stopApp(intent: String): Boolean = accessibilityService?.stopApp(intent) ?: false + + @JvmStatic + fun tap(x: Int, y: Int): Boolean { + if ((inputMethods and INPUT_METHOD_ACCESSIBILITY) == 0L) return false + return accessibilityService?.tap(x, y) ?: false + } + + @JvmStatic + fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, duration: Int): Boolean { + if ((inputMethods and INPUT_METHOD_ACCESSIBILITY) == 0L) return false + return accessibilityService?.swipe(x1, y1, x2, y2, duration) ?: false + } + + @JvmStatic + fun scroll(dx: Int, dy: Int): Boolean { + if ((inputMethods and INPUT_METHOD_ACCESSIBILITY) == 0L) return false + return accessibilityService?.scroll(dx, dy) ?: false + } + + @JvmStatic + fun keyEvent(key: Int, down: Boolean): Boolean { + if ((inputMethods and INPUT_METHOD_ACCESSIBILITY) == 0L) return false + return accessibilityService?.keyEvent(key, down) ?: false + } + + @JvmStatic + fun inputText(text: String): Boolean { + if ((inputMethods and INPUT_METHOD_ACCESSIBILITY) == 0L) return false + return accessibilityService?.inputText(text) ?: false + } + + @JvmStatic + fun screencap(): Bitmap? { + // Try MediaProjection first if enabled (faster and more reliable) + if ((screencapMethods and SCREENCAP_METHOD_MEDIA_PROJECTION) != 0L) { + mediaProjectionHolder?.capture()?.let { return it } + } + + // Fall back to Accessibility screenshot + if ((screencapMethods and SCREENCAP_METHOD_ACCESSIBILITY) != 0L) { + accessibilityService?.screencap()?.let { return it } + } + + return null + } + + @JvmStatic + fun getScreenSize(): IntArray { + return accessibilityService?.getScreenSize() ?: intArrayOf(0, 0) + } +} diff --git a/source/MaaAndroidControlUnit/java/gradle.properties b/source/MaaAndroidControlUnit/java/gradle.properties new file mode 100644 index 000000000..743ad0363 --- /dev/null +++ b/source/MaaAndroidControlUnit/java/gradle.properties @@ -0,0 +1,2 @@ +android.useAndroidX=true +kotlin.code.style=official diff --git a/source/MaaAndroidControlUnit/java/res/xml/accessibility_service_config.xml b/source/MaaAndroidControlUnit/java/res/xml/accessibility_service_config.xml new file mode 100644 index 000000000..31e19fab9 --- /dev/null +++ b/source/MaaAndroidControlUnit/java/res/xml/accessibility_service_config.xml @@ -0,0 +1,8 @@ + + diff --git a/source/MaaAndroidControlUnit/java/settings.gradle.kts b/source/MaaAndroidControlUnit/java/settings.gradle.kts new file mode 100644 index 000000000..943485255 --- /dev/null +++ b/source/MaaAndroidControlUnit/java/settings.gradle.kts @@ -0,0 +1,21 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } + plugins { + id("com.android.library") version "8.2.2" + id("org.jetbrains.kotlin.android") version "1.9.22" + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "MaaAndroidControlUnit" diff --git a/source/MaaFramework/API/MaaFramework.cpp b/source/MaaFramework/API/MaaFramework.cpp index 96c794aff..8b57a0f16 100644 --- a/source/MaaFramework/API/MaaFramework.cpp +++ b/source/MaaFramework/API/MaaFramework.cpp @@ -45,6 +45,29 @@ MaaController* MaaAdbControllerCreate( return new MAA_CTRL_NS::ControllerAgent(std::move(control_unit)); } +MaaController* MaaAndroidControllerCreate( + MaaAndroidScreencapMethod screencap_methods, + MaaAndroidInputMethod input_methods) +{ + LogFunc << VAR(screencap_methods) << VAR(input_methods); + +#ifndef __ANDROID__ + LogError << "This API" << __FUNCTION__ << "is only available on Android"; + std::ignore = screencap_methods; + std::ignore = input_methods; + return nullptr; +#else + auto control_unit = MAA_NS::AndroidControlUnitLibraryHolder::create_control_unit(screencap_methods, input_methods); + + if (!control_unit) { + LogError << "Failed to create control unit"; + return nullptr; + } + + return new MAA_CTRL_NS::ControllerAgent(std::move(control_unit)); +#endif +} + MaaController* MaaWin32ControllerCreate( void* hWnd, MaaWin32ScreencapMethod screencap_method, diff --git a/source/MaaFramework/CMakeLists.txt b/source/MaaFramework/CMakeLists.txt index 79a68378c..9033e3c64 100644 --- a/source/MaaFramework/CMakeLists.txt +++ b/source/MaaFramework/CMakeLists.txt @@ -20,6 +20,10 @@ if(WITH_ADB_CONTROLLER) add_dependencies(MaaFramework MaaAdbControlUnit) endif() +if(WITH_ANDROID_CONTROLLER) + add_dependencies(MaaFramework MaaAndroidControlUnit) +endif() + if(WITH_WIN32_CONTROLLER) add_dependencies(MaaFramework MaaWin32ControlUnit) endif() diff --git a/source/MaaPiCli/CLI/interactor.cpp b/source/MaaPiCli/CLI/interactor.cpp index a6cf2c0d2..c29da75df 100644 --- a/source/MaaPiCli/CLI/interactor.cpp +++ b/source/MaaPiCli/CLI/interactor.cpp @@ -194,6 +194,9 @@ void Interactor::print_config() const std::cout << MAA_NS::utf8_to_crt(std::format("\t\t{}\n", format_win32_config(config_.configuration().win32))); } break; + case InterfaceData::Controller::Type::Android: + std::cout << "\t\tNative Android Controller\n"; + break; default: LogError << "Unknown controller type" << VAR(config_.configuration().controller.type); break; @@ -363,6 +366,11 @@ void Interactor::select_controller() config_.configuration().controller.type = InterfaceData::Controller::Type::Win32; select_win32_hwnd(controller.win32); break; + case InterfaceData::Controller::Type::Android: + config_.configuration().controller.type = InterfaceData::Controller::Type::Android; + // Android controller does not require additional configuration + std::cout << "Using Native Android Controller\n\n"; + break; default: LogError << "Unknown controller type" << VAR(controller.type); break; diff --git a/source/MaaPiCli/Impl/Configurator.cpp b/source/MaaPiCli/Impl/Configurator.cpp index e83508db3..8bf6cc000 100644 --- a/source/MaaPiCli/Impl/Configurator.cpp +++ b/source/MaaPiCli/Impl/Configurator.cpp @@ -191,6 +191,15 @@ std::optional Configurator::generate_runtime() const runtime.controller_param = std::move(win32); } break; + case InterfaceData::Controller::Type::Android: { + RuntimeParam::AndroidParam android; + // Use default screencap and input methods + android.screencap = MaaAndroidScreencapMethod_Default; + android.input = MaaAndroidInputMethod_Accessibility; + + runtime.controller_param = std::move(android); + } break; + default: { LogError << "Unknown controller type" << VAR(controller.type); return std::nullopt; diff --git a/source/MaaPiCli/Impl/Runner.cpp b/source/MaaPiCli/Impl/Runner.cpp index 676d0185c..a23dcf64e 100644 --- a/source/MaaPiCli/Impl/Runner.cpp +++ b/source/MaaPiCli/Impl/Runner.cpp @@ -89,6 +89,9 @@ bool Runner::run(const RuntimeParam& param) controller_handle = MaaWin32ControllerCreate(p_win32_param->hwnd, p_win32_param->screencap, p_win32_param->mouse, p_win32_param->keyboard); } + else if (const auto* p_android_param = std::get_if(¶m.controller_param)) { + controller_handle = MaaAndroidControllerCreate(p_android_param->screencap, p_android_param->input); + } else { LogError << "Unknown controller type"; return false; diff --git a/source/include/ControlUnit/AndroidControlUnitAPI.h b/source/include/ControlUnit/AndroidControlUnitAPI.h new file mode 100644 index 000000000..4fe306d7c --- /dev/null +++ b/source/include/ControlUnit/AndroidControlUnitAPI.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include "ControlUnit/ControlUnitAPI.h" +#include "MaaFramework/MaaDef.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + + MAA_CONTROL_UNIT_API const char* MaaAndroidControlUnitGetVersion(); + + /** + * @brief Create Android control unit based on accessibility / media projection. + * + * @param screencap_methods bitmask of MaaAndroidScreencapMethod + * @param input_methods bitmask of MaaAndroidInputMethod + * @return MaaAndroidControlUnitHandle + */ + MAA_CONTROL_UNIT_API MaaAndroidControlUnitHandle + MaaAndroidControlUnitCreate(MaaAndroidScreencapMethod screencap_methods, MaaAndroidInputMethod input_methods); + + MAA_CONTROL_UNIT_API void MaaAndroidControlUnitDestroy(MaaAndroidControlUnitHandle handle); + +#ifdef __cplusplus +} +#endif diff --git a/source/include/ControlUnit/ControlUnitAPI.h b/source/include/ControlUnit/ControlUnitAPI.h index 65ffa3ef0..ab9d7e13c 100644 --- a/source/include/ControlUnit/ControlUnitAPI.h +++ b/source/include/ControlUnit/ControlUnitAPI.h @@ -2,6 +2,7 @@ #include #include +#include #include "Common/Conf.h" #include "MaaFramework/MaaDef.h" @@ -49,6 +50,12 @@ class AdbControlUnitAPI : public ControlUnitAPI virtual bool shell(const std::string& cmd, std::string& output) = 0; }; +class AndroidControlUnitAPI : public ControlUnitAPI +{ +public: + virtual ~AndroidControlUnitAPI() = default; +}; + class Win32ControlUnitAPI : public ControlUnitAPI { public: @@ -59,5 +66,6 @@ MAA_CTRL_UNIT_NS_END using MaaControlUnitHandle = MAA_CTRL_UNIT_NS::ControlUnitAPI*; using MaaAdbControlUnitHandle = MAA_CTRL_UNIT_NS::AdbControlUnitAPI*; +using MaaAndroidControlUnitHandle = MAA_CTRL_UNIT_NS::AndroidControlUnitAPI*; using MaaWin32ControlUnitHandle = MAA_CTRL_UNIT_NS::Win32ControlUnitAPI*; using MaaCustomControlUnitHandle = MAA_CTRL_UNIT_NS::ControlUnitAPI*; diff --git a/source/include/LibraryHolder/ControlUnit.h b/source/include/LibraryHolder/ControlUnit.h index c78345745..ac67c87a3 100644 --- a/source/include/LibraryHolder/ControlUnit.h +++ b/source/include/LibraryHolder/ControlUnit.h @@ -11,6 +11,7 @@ MAA_CTRL_UNIT_NS_BEGIN class ControlUnitAPI; class AdbControlUnitAPI; +class AndroidControlUnitAPI; class Win32ControlUnitAPI; class CustomControlUnitAPI; MAA_CTRL_UNIT_NS_END @@ -51,6 +52,19 @@ class Win32ControlUnitLibraryHolder : public LibraryHolder +{ +public: + static std::shared_ptr + create_control_unit(MaaAndroidScreencapMethod screencap_methods, MaaAndroidInputMethod input_methods); + +private: + inline static const std::filesystem::path libname_ = MAA_NS::path("MaaAndroidControlUnit"); + inline static const std::string version_func_name_ = "MaaAndroidControlUnitGetVersion"; + inline static const std::string create_func_name_ = "MaaAndroidControlUnitCreate"; + inline static const std::string destroy_func_name_ = "MaaAndroidControlUnitDestroy"; +}; + class DbgControlUnitLibraryHolder : public LibraryHolder { public: diff --git a/source/include/ProjectInterface/Types.h b/source/include/ProjectInterface/Types.h index 391453485..1e4cd7535 100644 --- a/source/include/ProjectInterface/Types.h +++ b/source/include/ProjectInterface/Types.h @@ -33,6 +33,7 @@ struct InterfaceData { Invalid, Adb, + Android, Win32, }; @@ -288,6 +289,12 @@ struct RuntimeParam MaaWin32InputMethod keyboard = MaaWin32InputMethod_None; }; + struct AndroidParam + { + MaaAndroidScreencapMethod screencap = MaaAndroidScreencapMethod_Default; + MaaAndroidInputMethod input = MaaAndroidInputMethod_Accessibility; + }; + struct Task { std::string name; @@ -303,7 +310,7 @@ struct RuntimeParam std::filesystem::path cwd; }; - std::variant controller_param; + std::variant controller_param; std::vector resource_path; std::vector task; diff --git a/source/modules/MaaFramework.cppm b/source/modules/MaaFramework.cppm index a12e5fb5c..f82c9d9d8 100644 --- a/source/modules/MaaFramework.cppm +++ b/source/modules/MaaFramework.cppm @@ -76,6 +76,19 @@ export constexpr auto _MaaAdbInputMethod_None = MaaAdbInputMethod_None; export constexpr auto _MaaAdbInputMethod_All = MaaAdbInputMethod_All; export constexpr auto _MaaAdbInputMethod_Default = MaaAdbInputMethod_Default; +export using ::MaaAndroidScreencapMethod; +export constexpr auto _MaaAndroidScreencapMethod_None = MaaAndroidScreencapMethod_None; +export constexpr auto _MaaAndroidScreencapMethod_AccessibilityScreenshot = MaaAndroidScreencapMethod_AccessibilityScreenshot; +export constexpr auto _MaaAndroidScreencapMethod_MediaProjection = MaaAndroidScreencapMethod_MediaProjection; +export constexpr auto _MaaAndroidScreencapMethod_All = MaaAndroidScreencapMethod_All; +export constexpr auto _MaaAndroidScreencapMethod_Default = MaaAndroidScreencapMethod_Default; + +export using ::MaaAndroidInputMethod; +export constexpr auto _MaaAndroidInputMethod_None = MaaAndroidInputMethod_None; +export constexpr auto _MaaAndroidInputMethod_Accessibility = MaaAndroidInputMethod_Accessibility; +export constexpr auto _MaaAndroidInputMethod_All = MaaAndroidInputMethod_All; +export constexpr auto _MaaAndroidInputMethod_Default = MaaAndroidInputMethod_Default; + export using ::MaaWin32ScreencapMethod; export constexpr auto _MaaWin32ScreencapMethod_None = MaaWin32ScreencapMethod_None; export constexpr auto _MaaWin32ScreencapMethod_GDI = MaaWin32ScreencapMethod_GDI; @@ -113,6 +126,7 @@ export using ::MaaContextClone; export using ::MaaCustomControllerCallbacks; export using ::MaaAdbControllerCreate; +export using ::MaaAndroidControllerCreate; export using ::MaaWin32ControllerCreate; export using ::MaaCustomControllerCreate; export using ::MaaDbgControllerCreate;