Skip to content

Commit 22705ea

Browse files
thymikeethymikee
committed
fix: harden android fill fallback strategy
1 parent 5e35437 commit 22705ea

2 files changed

Lines changed: 99 additions & 12 deletions

File tree

‎src/platforms/android/__tests__/index.test.ts‎

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { promises as fs } from 'node:fs';
44
import os from 'node:os';
55
import path from 'node:path';
66
import {
7+
fillAndroid,
78
inferAndroidAppName,
89
isAmStartError,
910
listAndroidApps,
@@ -737,7 +738,7 @@ test('typeAndroid uses adb input text for ascii text', async () => {
737738
);
738739
});
739740

740-
test('typeAndroid URL-encodes shell-sensitive ascii text', async () => {
741+
test('typeAndroid passes shell-sensitive ascii text to adb input text', async () => {
741742
await withMockedAdb(
742743
'agent-device-android-type-ascii-special-',
743744
'#!/bin/sh\nprintf "%s\\n" "$@" > "$AGENT_DEVICE_TEST_ARGS_FILE"\nexit 0\n',
@@ -753,13 +754,13 @@ test('typeAndroid URL-encodes shell-sensitive ascii text', async () => {
753754
'shell',
754755
'input',
755756
'text',
756-
'curtis.layne%2Btest%2B73kmc%40uber.com',
757+
'curtis.layne+test+73kmc@uber.com',
757758
]);
758759
},
759760
);
760761
});
761762

762-
test('typeAndroid URL-encodes percent signs', async () => {
763+
test('typeAndroid preserves percent signs while encoding spaces', async () => {
763764
await withMockedAdb(
764765
'agent-device-android-type-ascii-percent-',
765766
'#!/bin/sh\nprintf "%s\\n" "$@" > "$AGENT_DEVICE_TEST_ARGS_FILE"\nexit 0\n',
@@ -775,12 +776,77 @@ test('typeAndroid URL-encodes percent signs', async () => {
775776
'shell',
776777
'input',
777778
'text',
778-
'50%25%scomplete',
779+
'50%%scomplete',
779780
]);
780781
},
781782
);
782783
});
783784

785+
test('fillAndroid falls back to clipboard paste when adb input text truncates', async () => {
786+
await withMockedAdb(
787+
'agent-device-android-fill-fallback-',
788+
[
789+
'#!/bin/sh',
790+
'STATE_FILE="$(dirname "$AGENT_DEVICE_TEST_ARGS_FILE")/fill_state.txt"',
791+
'CLIP_FILE="$(dirname "$AGENT_DEVICE_TEST_ARGS_FILE")/clipboard_state.txt"',
792+
'printf "__CMD__\\n" >> "$AGENT_DEVICE_TEST_ARGS_FILE"',
793+
'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"',
794+
'if [ "$1" = "-s" ]; then',
795+
' shift',
796+
' shift',
797+
'fi',
798+
'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "tap" ]; then',
799+
' exit 0',
800+
'fi',
801+
'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "keyevent" ] && [ "$4" = "KEYCODE_MOVE_END" ]; then',
802+
' exit 0',
803+
'fi',
804+
'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "keyevent" ] && [ "$4" = "KEYCODE_DEL" ]; then',
805+
' : > "$STATE_FILE"',
806+
' exit 0',
807+
'fi',
808+
'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "text" ]; then',
809+
' # Simulate WebView truncation on shell text input with special chars.',
810+
' if [ "$4" = "curtis.layne+test+73kmc@uber.com" ]; then',
811+
' printf "curti" > "$STATE_FILE"',
812+
' else',
813+
' printf "%s" "$4" > "$STATE_FILE"',
814+
' fi',
815+
' exit 0',
816+
'fi',
817+
'if [ "$1" = "shell" ] && [ "$2" = "cmd" ] && [ "$3" = "clipboard" ] && [ "$4" = "set" ] && [ "$5" = "text" ]; then',
818+
' printf "%s" "$6" > "$CLIP_FILE"',
819+
' exit 0',
820+
'fi',
821+
'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "keyevent" ] && [ "$4" = "KEYCODE_PASTE" ]; then',
822+
' cat "$CLIP_FILE" > "$STATE_FILE"',
823+
' exit 0',
824+
'fi',
825+
'if [ "$1" = "shell" ] && [ "$2" = "input" ] && [ "$3" = "keyevent" ] && [ "$4" = "279" ]; then',
826+
' cat "$CLIP_FILE" > "$STATE_FILE"',
827+
' exit 0',
828+
'fi',
829+
'if [ "$1" = "exec-out" ] && [ "$2" = "uiautomator" ] && [ "$3" = "dump" ] && [ "$4" = "/dev/tty" ]; then',
830+
' text="$(cat "$STATE_FILE" 2>/dev/null)"',
831+
' printf "<?xml version=\\"1.0\\" encoding=\\"UTF-8\\"?><hierarchy><node class=\\"android.widget.EditText\\" text=\\"%s\\" focused=\\"true\\" bounds=\\"[0,0][200,100]\\"/></hierarchy>" "$text"',
832+
' exit 0',
833+
'fi',
834+
'echo "unexpected args: $@" >&2',
835+
'exit 1',
836+
'',
837+
].join('\n'),
838+
async ({ argsLogPath, device }) => {
839+
await fillAndroid(device, 10, 10, 'curtis.layne+test+73kmc@uber.com');
840+
const logged = await fs.readFile(argsLogPath, 'utf8');
841+
assert.match(logged, /shell\ninput\ntext\ncurtis\.layne\+test\+73kmc@uber\.com/);
842+
assert.match(logged, /shell\ncmd\nclipboard\nset\ntext\ncurtis\.layne\+test\+73kmc@uber\.com/);
843+
assert.match(logged, /shell\ninput\nkeyevent\nKEYCODE_PASTE/);
844+
const shellInputTextCount = (logged.match(/shell\ninput\ntext\n/g) ?? []).length;
845+
assert.equal(shellInputTextCount, 1);
846+
},
847+
);
848+
});
849+
784850
test('typeAndroid reports clear error when unicode input is unsupported', async () => {
785851
await withMockedAdb(
786852
'agent-device-android-type-unicode-unsupported-',

‎src/platforms/android/index.ts‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -537,29 +537,50 @@ export async function fillAndroid(
537537
text: string,
538538
): Promise<void> {
539539
const textCodePointLength = Array.from(text).length;
540-
const attempts = [
541-
{ clearPadding: 12, minClear: 8, maxClear: 48, chunkSize: 4, delayMs: 0 },
542-
{ clearPadding: 24, minClear: 16, maxClear: 96, chunkSize: 1, delayMs: 15 },
543-
] as const;
540+
const usesClipboardForText = shouldUseClipboardTextInjection(text);
541+
const attempts: Array<{
542+
strategy: 'input_text' | 'clipboard_paste' | 'chunked_input';
543+
clearPadding: number;
544+
minClear: number;
545+
maxClear: number;
546+
}> = [
547+
{ strategy: 'input_text', clearPadding: 12, minClear: 8, maxClear: 48 },
548+
{ strategy: 'clipboard_paste', clearPadding: 12, minClear: 8, maxClear: 48 },
549+
];
550+
if (!usesClipboardForText) {
551+
attempts.push({ strategy: 'chunked_input', clearPadding: 24, minClear: 16, maxClear: 96 });
552+
}
544553

545-
await focusAndroid(device, x, y);
546554
let lastActual: string | null = null;
555+
const attemptedStrategies: string[] = [];
547556

548557
for (const attempt of attempts) {
558+
await focusAndroid(device, x, y);
549559
const clearCount = clampCount(
550560
textCodePointLength + attempt.clearPadding,
551561
attempt.minClear,
552562
attempt.maxClear,
553563
);
554564
await clearFocusedText(device, clearCount);
555-
await typeAndroidChunked(device, text, attempt.chunkSize, attempt.delayMs);
565+
attemptedStrategies.push(attempt.strategy);
566+
if (attempt.strategy === 'input_text') {
567+
await typeAndroid(device, text);
568+
} else if (attempt.strategy === 'clipboard_paste') {
569+
const clipboardResult = await typeAndroidViaClipboard(device, text);
570+
if (clipboardResult !== 'ok') {
571+
continue;
572+
}
573+
} else {
574+
await typeAndroidChunked(device, text, 1, 15);
575+
}
556576
lastActual = await readInputValueAtPoint(device, x, y);
557577
if (lastActual === text) return;
558578
}
559579

560580
throw new AppError('COMMAND_FAILED', 'Android fill verification failed', {
561581
expected: text,
562582
actual: lastActual ?? null,
583+
strategies: attemptedStrategies,
563584
});
564585
}
565586

@@ -1124,8 +1145,8 @@ function shouldUseClipboardTextInjection(text: string): boolean {
11241145
}
11251146

11261147
function encodeAndroidInputText(text: string): string {
1127-
// Android shell input understands URL-escaped bytes and `%s` as a space token.
1128-
return encodeURIComponent(text).replace(/%20/g, '%s');
1148+
// Android shell input uses `%s` as the escaped token for spaces.
1149+
return text.replace(/ /g, '%s');
11291150
}
11301151

11311152
async function typeAndroidViaClipboard(

0 commit comments

Comments
 (0)