fix: harden android non-ascii text input and document IME workaround (#118)

Author: thymikee

README.md

@@ -287,6 +287,13 @@ Android fill reliability:
 - `fill` now verifies the entered value on Android.
 - If value does not match, agent-device clears the field and retries once with slower typing.
 - This reduces IME-related character swaps on long strings (e.g. emails and IDs).
+- Some Android system images cannot inject non-ASCII text (for example Chinese or emoji) through shell input.
+- If this occurs, install an ADB keyboard IME from a trusted source, verify checksum/signature, and enable it only for test sessions:
+  - Trusted sources: https://github.com/senzhk/ADBKeyBoard or https://f-droid.org/packages/com.android.adbkeyboard/
+  - `adb -s <serial> install <path-to-adbkeyboard.apk>`
+  - `adb -s <serial> shell ime enable com.android.adbkeyboard/.AdbIME`
+  - `adb -s <serial> shell ime set com.android.adbkeyboard/.AdbIME`
+  - `adb -s <serial> shell ime list -s` (verify current/default IME)
 
 Settings helpers:
 - `settings wifi on|off`

skills/agent-device/SKILL.md

@@ -111,6 +111,7 @@ agent-device batch --steps-file /tmp/batch-steps.json --json
 - Permission settings are app-scoped and require an active session app:
   `settings permission <grant|deny|reset> <camera|microphone|photos|contacts|notifications> [full|limited]`
 - `full|limited` mode applies only to iOS `photos`; other targets reject mode.
+- On Android, non-ASCII `fill/type` may require an ADB keyboard IME on some system images; only install IME APKs from trusted sources and verify checksum/signature.
 - If using `--save-script`, prefer explicit path syntax (`--save-script=flow.ad` or `./flow.ad`).
 
 ## Security and Trust Notes

src/platforms/android/__tests__/index.test.ts

@@ -11,6 +11,7 @@ import {
   parseAndroidLaunchComponent,
   setAndroidSetting,
   swipeAndroid,
+  typeAndroid,
 } from '../index.ts';
 import type { DeviceInfo } from '../../../utils/device.ts';
 import { AppError } from '../../../utils/errors.ts';
@@ -425,6 +426,97 @@ test('parseAndroidLaunchComponent handles multi-entry resolve output', () => {
   );
 });
 
+test('typeAndroid uses clipboard paste for unicode text', async () => {
+  await withMockedAdb(
+    'agent-device-android-type-unicode-',
+    [
+      '#!/bin/sh',
+      'printf "__CMD__\\n" >> "$AGENT_DEVICE_TEST_ARGS_FILE"',
+      'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"',
+      'if [ "$1" = "-s" ]; then',
+      '  shift',
+      '  shift',
+      'fi',
+      'if [ "$1" = "shell" ] && [ "$2" = "cmd" ] && [ "$3" = "clipboard" ] && [ "$4" = "set" ] && [ "$5" = "text" ]; then',
+      '  exit 0',
+      'fi',
+      'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "keyevent" ] && [ "$4" = "KEYCODE_PASTE" ]; then',
+      '  exit 0',
+      'fi',
+      'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "text" ]; then',
+      '  echo "unexpected fallback to input text" >&2',
+      '  exit 1',
+      'fi',
+      'exit 1',
+      '',
+    ].join('\n'),
+    async ({ argsLogPath, device }) => {
+      await typeAndroid(device, '很 ☝ 😀');
+      const logged = await fs.readFile(argsLogPath, 'utf8');
+      assert.match(logged, /shell\ncmd\nclipboard\nset\ntext\n很 ☝ 😀/);
+      assert.match(logged, /shell\ninput\nkeyevent\nKEYCODE_PASTE/);
+      assert.doesNotMatch(logged, /shell\ninput\ntext/);
+    },
+  );
+});
+
+test('typeAndroid uses adb input text for ascii text', async () => {
+  await withMockedAdb(
+    'agent-device-android-type-ascii-',
+    '#!/bin/sh\nprintf "%s\\n" "$@" > "$AGENT_DEVICE_TEST_ARGS_FILE"\nexit 0\n',
+    async ({ argsLogPath, device }) => {
+      await typeAndroid(device, 'hello world');
+      const args = (await fs.readFile(argsLogPath, 'utf8'))
+        .trim()
+        .split('\n')
+        .filter(Boolean);
+      assert.deepEqual(args, [
+        '-s',
+        'emulator-5554',
+        'shell',
+        'input',
+        'text',
+        'hello%sworld',
+      ]);
+    },
+  );
+});
+
+test('typeAndroid reports clear error when unicode input is unsupported', async () => {
+  await withMockedAdb(
+    'agent-device-android-type-unicode-unsupported-',
+    [
+      '#!/bin/sh',
+      'if [ "$1" = "-s" ]; then',
+      '  shift',
+      '  shift',
+      'fi',
+      'if [ "$1" = "shell" ] && [ "$2" = "cmd" ] && [ "$3" = "clipboard" ] && [ "$4" = "set" ] && [ "$5" = "text" ]; then',
+      '  echo "No shell command implementation."',
+      '  exit 0',
+      'fi',
+      'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "text" ]; then',
+      "  echo \"Exception occurred while executing 'text':\" >&2",
+      '  echo "java.lang.NullPointerException" >&2',
+      '  exit 255',
+      'fi',
+      'echo "unexpected args: $@" >&2',
+      'exit 1',
+      '',
+    ].join('\n'),
+    async ({ device }) => {
+      await assert.rejects(
+        () => typeAndroid(device, '很'),
+        (error: unknown) => {
+          assert.equal(error instanceof AppError, true);
+          assert.equal((error as AppError).code, 'COMMAND_FAILED');
+          assert.match((error as AppError).message, /non-ascii text input is not supported/i);
+          return true;
+        },
+      );
+    },
+  );
+});
 test('setAndroidSetting permission grant camera uses pm grant', async () => {
   await withMockedAdb(
     'agent-device-android-permission-camera-',

src/platforms/android/index.ts

@@ -446,8 +446,24 @@ export async function longPressAndroid(
 }
 
 export async function typeAndroid(device: DeviceInfo, text: string): Promise<void> {
-  const encoded = text.replace(/ /g, '%s');
-  await runCmd('adb', adbArgs(device, ['shell', 'input', 'text', encoded]));
+  if (shouldUseClipboardTextInjection(text)) {
+    const clipboardResult = await typeAndroidViaClipboard(device, text);
+    if (clipboardResult === 'ok') return;
+  }
+  try {
+    const encoded = text.replace(/ /g, '%s');
+    await runCmd('adb', adbArgs(device, ['shell', 'input', 'text', encoded]));
+  } catch (error) {
+    if (shouldUseClipboardTextInjection(text) && isAndroidInputTextUnsupported(error)) {
+      throw new AppError(
+        'COMMAND_FAILED',
+        'Non-ASCII text input is not supported on this Android shell. Install an ADB keyboard IME or use ASCII input.',
+        { textPreview: text.slice(0, 32) },
+        error instanceof Error ? error : undefined,
+      );
+    }
+    throw error;
+  }
 }
 
 export async function focusAndroid(device: DeviceInfo, x: number, y: number): Promise<void> {
@@ -460,6 +476,7 @@ export async function fillAndroid(
   y: number,
   text: string,
 ): Promise<void> {
+  const textCodePointLength = Array.from(text).length;
   const attempts = [
     { clearPadding: 12, minClear: 8, maxClear: 48, chunkSize: 4, delayMs: 0 },
     { clearPadding: 24, minClear: 16, maxClear: 96, chunkSize: 1, delayMs: 15 },
@@ -470,7 +487,7 @@ export async function fillAndroid(
 
   for (const attempt of attempts) {
     const clearCount = clampCount(
-      text.length + attempt.clearPadding,
+      textCodePointLength + attempt.clearPadding,
       attempt.minClear,
       attempt.maxClear,
     );
@@ -845,15 +862,64 @@ async function typeAndroidChunked(
   delayMs: number,
 ): Promise<void> {
   const size = Math.max(1, Math.floor(chunkSize));
-  for (let i = 0; i < text.length; i += size) {
-    const chunk = text.slice(i, i + size);
+  const chars = Array.from(text);
+  for (let i = 0; i < chars.length; i += size) {
+    const chunk = chars.slice(i, i + size).join('');
     await typeAndroid(device, chunk);
-    if (delayMs > 0 && i + size < text.length) {
+    if (delayMs > 0 && i + size < chars.length) {
       await sleep(delayMs);
     }
   }
 }
 
+function shouldUseClipboardTextInjection(text: string): boolean {
+  for (const char of text) {
+    const code = char.codePointAt(0);
+    if (code === undefined) continue;
+    if (code < 0x20 || code > 0x7e) return true;
+  }
+  return false;
+}
+
+async function typeAndroidViaClipboard(
+  device: DeviceInfo,
+  text: string,
+): Promise<'ok' | 'unsupported' | 'failed'> {
+  const setClipboard = await runCmd(
+    'adb',
+    adbArgs(device, ['shell', 'cmd', 'clipboard', 'set', 'text', text]),
+    { allowFailure: true },
+  );
+  if (setClipboard.exitCode !== 0) return 'failed';
+  if (isClipboardShellUnsupported(setClipboard.stdout, setClipboard.stderr)) return 'unsupported';
+
+  const pasteByName = await runCmd(
+    'adb',
+    adbArgs(device, ['shell', 'input', 'keyevent', 'KEYCODE_PASTE']),
+    { allowFailure: true },
+  );
+  if (pasteByName.exitCode === 0) return 'ok';
+
+  const pasteByCode = await runCmd('adb', adbArgs(device, ['shell', 'input', 'keyevent', '279']), {
+    allowFailure: true,
+  });
+  return pasteByCode.exitCode === 0 ? 'ok' : 'failed';
+}
+
+function isClipboardShellUnsupported(stdout: string, stderr: string): boolean {
+  const haystack = `${stdout}\n${stderr}`.toLowerCase();
+  return haystack.includes('no shell command implementation') || haystack.includes('unknown command');
+}
+
+function isAndroidInputTextUnsupported(error: unknown): boolean {
+  if (!(error instanceof AppError)) return false;
+  if (error.code !== 'COMMAND_FAILED') return false;
+  const stderr = String((error.details as any)?.stderr ?? '').toLowerCase();
+  if (stderr.includes("exception occurred while executing 'text'")) return true;
+  if (stderr.includes('nullpointerexception') && stderr.includes('inputshellcommand.sendtext')) return true;
+  return false;
+}
+
 async function clearFocusedText(device: DeviceInfo, count: number): Promise<void> {
   const deletes = Math.max(0, count);
   await runCmd('adb', adbArgs(device, ['shell', 'input', 'keyevent', 'KEYCODE_MOVE_END']), {

website/docs/docs/commands.md

@@ -66,6 +66,7 @@ agent-device pinch 0.5 200 400 # zoom out at coordinates (iOS simulator)
 
 `fill` clears then types. `type` does not clear.
 On Android, `fill` also verifies text and performs one clear-and-retry pass on mismatch.
+Some Android images cannot enter non-ASCII text over shell input; in that case use a trusted ADB keyboard IME and verify APK checksum/signature before install.
 `swipe` accepts an optional `durationMs` argument (default `250ms`, range `16..10000`).
 On iOS, swipe duration is clamped to a safe range (`16..60ms`) to avoid longpress side effects.
 `scrollintoview` accepts plain text or a snapshot ref (`@eN`); ref mode uses best-effort geometry-based scrolling without post-scroll verification. Run `snapshot` again before follow-up `@ref` commands.

website/docs/docs/known-limitations.md

@@ -19,3 +19,22 @@ This is an Apple platform constraint that affects all XCUITest-based automation
   echo "some text" | xcrun simctl pbcopy booted
   ```
 - **Test the dialog manually** — the "Allow Paste" UX cannot be exercised through XCUITest-based automation.
+
+## Android: non-ASCII text over `adb shell input text`
+
+Some Android system images fail to inject non-ASCII text (for example Chinese characters or emoji) through `adb shell input text`.
+
+**Workarounds:**
+
+- **Use an ADB keyboard IME for test runs**:
+  ```bash
+  adb -s <serial> install <path-to-adbkeyboard.apk>
+  adb -s <serial> shell ime enable com.android.adbkeyboard/.AdbIME
+  adb -s <serial> shell ime set com.android.adbkeyboard/.AdbIME
+  ```
+- **Use trusted APK sources only** (official project: https://github.com/senzhk/ADBKeyBoard or F-Droid: https://f-droid.org/packages/com.android.adbkeyboard/), and verify checksum/signature before installing.
+- **Revert to your normal IME after automation**:
+  ```bash
+  adb -s <serial> shell ime list -s
+  adb -s <serial> shell ime set <previous-ime-id>
+  ```