fix: stabilize Android replay interactions

Author: thymikee

src/platforms/android/__tests__/index.test.ts

@@ -1720,6 +1720,35 @@ test('getAndroidKeyboardState uses latest visibility value when dumpsys contains
   );
 });
 
+test('getAndroidKeyboardState treats stale input view as hidden when the IME window is hidden', async () => {
+  await withMockedAdb(
+    'agent-device-android-keyboard-stale-input-view-',
+    [
+      '#!/bin/sh',
+      'if [ "$1" = "-s" ]; then',
+      '  shift',
+      '  shift',
+      'fi',
+      'if [ "$1" = "shell" ] && [ "$2" = "dumpsys" ] && [ "$3" = "input_method" ]; then',
+      '  echo "mInputShown=false"',
+      '  echo "mDecorViewVisible=false mWindowVisible=false mInShowWindow=false"',
+      '  echo "mIsInputViewShown=true"',
+      '  echo "inputType=0x21"',
+      '  exit 0',
+      'fi',
+      'echo "unexpected args: $@" >&2',
+      'exit 1',
+      '',
+    ].join('\n'),
+    async ({ device }) => {
+      const state = await getAndroidKeyboardState(device);
+      assert.equal(state.visible, false);
+      assert.equal(state.inputType, '0x21');
+      assert.equal(state.type, 'email');
+    },
+  );
+});
+
 test('dismissAndroidKeyboard skips keyevent when keyboard is already hidden', async () => {
   await withMockedAdb(
     'agent-device-android-keyboard-dismiss-hidden-',

src/platforms/android/device-input-state.ts

@@ -185,18 +185,28 @@ function parseLastDumpsysValue(stdout: string, pattern: RegExp): string | undefi
 
 function parseAndroidKeyboardVisibility(stdout: string): boolean | null {
   const latestByKey = new Map<string, boolean>();
-  const pattern = /\b(mInputShown|mIsInputViewShown|isInputViewShown)=([a-zA-Z]+)\b/g;
+  const pattern =
+    /\b(mInputShown|mIsInputViewShown|isInputViewShown|mDecorViewVisible|mWindowVisible|mInShowWindow)=([a-zA-Z]+)\b/g;
   for (const match of stdout.matchAll(pattern)) {
     const key = match[1];
     const value = match[2]?.toLowerCase();
     if (!key || (value !== 'true' && value !== 'false')) continue;
     latestByKey.set(key, value === 'true');
   }
   if (latestByKey.size === 0) return null;
-  for (const visible of latestByKey.values()) {
-    if (visible) return true;
-  }
-  return false;
+
+  const windowVisible =
+    latestByKey.get('mWindowVisible') ??
+    latestByKey.get('mDecorViewVisible') ??
+    latestByKey.get('mInShowWindow');
+  if (windowVisible !== undefined) return windowVisible;
+
+  const inputShown = latestByKey.get('mInputShown');
+  if (inputShown !== undefined) return inputShown;
+
+  const inputViewShown =
+    latestByKey.get('mIsInputViewShown') ?? latestByKey.get('isInputViewShown');
+  return inputViewShown ?? null;
 }
 
 function classifyAndroidKeyboardType(inputType: string): AndroidKeyboardType {

src/platforms/android/fill-verification.ts

@@ -11,7 +11,7 @@ import {
 import { sleep } from './adb.ts';
 import { getAndroidKeyboardState } from './device-input-state.ts';
 import { isAndroidInputMethodOwnedNode } from './input-ownership.ts';
-import { dumpUiHierarchy } from './snapshot.ts';
+import { captureAndroidUiHierarchyXml } from './snapshot.ts';
 import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts';
 
 export type AndroidFillVerificationNode = FillDiagnosticNode & {
@@ -90,7 +90,7 @@ export async function readAndroidTextAtPoint(
   x: number,
   y: number,
 ): Promise<string | null> {
-  return readAndroidTextAtPointInHierarchy(await dumpUiHierarchy(device), x, y);
+  return readAndroidTextAtPointInHierarchy(await captureAndroidUiHierarchyXml(device), x, y);
 }
 
 export function verifyAndroidFilledTextInHierarchy(
@@ -154,7 +154,13 @@ async function inspectAndroidFilledText(
   expected: string,
   context: AndroidFillVerificationContext,
 ): Promise<AndroidFillVerification> {
-  return verifyAndroidFilledTextInHierarchy(await dumpUiHierarchy(device), x, y, expected, context);
+  return verifyAndroidFilledTextInHierarchy(
+    await captureAndroidUiHierarchyXml(device),
+    x,
+    y,
+    expected,
+    context,
+  );
 }
 
 function inspectAndroidTextAtPointInHierarchy(

src/platforms/android/input-actions.ts

@@ -6,6 +6,8 @@ import { buildScrollGesturePlan, type ScrollDirection } from '../../core/scroll-
 import { runAndroidAdb, sleep } from './adb.ts';
 import { resolveAndroidTextInjector } from './adb-executor.ts';
 import { getAndroidKeyboardState, type AndroidKeyboardState } from './device-input-state.ts';
+import { captureAndroidUiHierarchyXml } from './snapshot.ts';
+import { androidUiNodes } from './ui-hierarchy.ts';
 import {
   androidFillFailureDetails,
   androidFillFailureMessage,
@@ -206,7 +208,7 @@ export async function scrollAndroid(
   direction: ScrollDirection,
   options?: { amount?: number; pixels?: number },
 ): Promise<Record<string, unknown>> {
-  const size = await getAndroidScreenSize(device);
+  const size = await getAndroidGestureViewportSize(device);
   const plan = buildScrollGesturePlan({
     direction,
     amount: options?.amount,
@@ -290,6 +292,38 @@ export async function getAndroidScreenSize(
   return { width: Number(match[1]), height: Number(match[2]) };
 }
 
+async function getAndroidGestureViewportSize(
+  device: DeviceInfo,
+): Promise<{ width: number; height: number }> {
+  try {
+    const xml = await captureAndroidUiHierarchyXml(device);
+    const viewport = largestAndroidUiNodeRect(xml);
+    if (viewport) return viewport;
+  } catch (error) {
+    emitDiagnostic({
+      level: 'warn',
+      phase: 'android_gesture_viewport_probe_failed',
+      data: {
+        error: error instanceof Error ? error.message : String(error),
+      },
+    });
+  }
+  return await getAndroidScreenSize(device);
+}
+
+function largestAndroidUiNodeRect(xml: string): { width: number; height: number } | null {
+  let largest: { width: number; height: number; area: number } | null = null;
+  for (const node of androidUiNodes(xml)) {
+    const rect = node.rect;
+    if (!rect || rect.width <= 0 || rect.height <= 0) continue;
+    const area = rect.width * rect.height;
+    if (!largest || area > largest.area) {
+      largest = { width: rect.x + rect.width, height: rect.y + rect.height, area };
+    }
+  }
+  return largest ? { width: largest.width, height: largest.height } : null;
+}
+
 const ANDROID_INPUT_TEXT_CHUNK_SIZE = 8;
 
 async function typeAndroidShell(

src/platforms/android/snapshot.ts

@@ -60,14 +60,22 @@ const RETRYABLE_ADB_STDERR_PATTERNS = [
   'no such file or directory',
 ] as const;
 
-type AndroidSnapshotOptions = SnapshotOptions & {
+export type AndroidSnapshotOptions = SnapshotOptions & {
   helperArtifact?: AndroidSnapshotHelperArtifact;
   helperInstallPolicy?: AndroidSnapshotHelperInstallPolicy;
   helperAdb?: AndroidAdbExecutor | AndroidAdbProvider;
   helperWaitForIdleTimeoutMs?: number;
   includeHiddenContentHints?: boolean;
 };
 
+export async function captureAndroidUiHierarchyXml(
+  device: DeviceInfo,
+  options: AndroidSnapshotOptions = {},
+): Promise<string> {
+  const adb = resolveAndroidAdbProvider(device, options.helperAdb).exec;
+  return (await captureAndroidUiHierarchy(device, options, adb)).xml;
+}
+
 export async function snapshotAndroid(
   device: DeviceInfo,
   options: AndroidSnapshotOptions = {},